Merge branch '小区周边POI定时任务' into script_develop
# Conflicts: # projects/fangjia-script/pom.xml # projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy # projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy
This commit is contained in:
@@ -39,13 +39,15 @@ spec:
|
||||
name: web
|
||||
protocol: TCP
|
||||
readinessProbe:
|
||||
tcpSocket:
|
||||
port: 8080
|
||||
httpGet:
|
||||
path: /k8s/readiness
|
||||
port: web
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
tcpSocket:
|
||||
port: 8080
|
||||
httpGet:
|
||||
path: /k8s/liveness
|
||||
port: web
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 5
|
||||
volumeMounts:
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
package com.fangjia.bkoff.controllers;
|
||||
|
||||
/**
|
||||
* 房价网后台存活监控
|
||||
*
|
||||
* @author shenjian
|
||||
* @since 2021/6/7
|
||||
*/
|
||||
public class K8sController {
|
||||
|
||||
/**
|
||||
* 存活检查
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
public Object livenessAction() {
|
||||
return "liveness";
|
||||
}
|
||||
|
||||
/**
|
||||
* 就绪检查
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
public Object readinessAction() {
|
||||
return "readiness";
|
||||
}
|
||||
}
|
||||
+2
-1
@@ -40,7 +40,8 @@ public class AuthInterceptor implements Interceptor {
|
||||
request.setAttribute("__resources", request.getSession().getAttribute(userId + "_resources"));
|
||||
|
||||
if (invocation.getAction().getController().getName().equals("login")
|
||||
|| invocation.getAction().getController().getName().equals("")) {
|
||||
|| invocation.getAction().getController().getName().equals("")
|
||||
|| invocation.getAction().getController().getName().equals("k8s")) {
|
||||
return invocation.invoke();
|
||||
}
|
||||
|
||||
|
||||
+3
-2
@@ -237,7 +237,8 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl<Seawater> i
|
||||
}
|
||||
|
||||
private JSONObject query(String id,String category, String city, String region, String name, List<String> alias,
|
||||
List<String> regionAlias, Integer startDate, Integer endDate, Double startArea, Double endArea, Integer startPrice, Integer endPrice, Integer room, String root,String sort, boolean order, int start, int limit,String table){
|
||||
List<String> regionAlias, Integer startDate, Integer endDate, Double startArea, Double endArea, Integer startPrice, Integer endPrice,
|
||||
Integer room, String root,String sort, boolean order, int start, int limit,String table){
|
||||
long time1 = System.currentTimeMillis();
|
||||
JSONObject reqBody = new JSONObject();
|
||||
JSONObject q = new JSONObject();
|
||||
@@ -351,7 +352,7 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl<Seawater> i
|
||||
if(endPrice != null){
|
||||
temp2.put("lte", endPrice);
|
||||
}
|
||||
param.put("total_price", temp2);
|
||||
param.put("avg_price", temp2);
|
||||
temp.put("range", param);
|
||||
jsonList.add(temp);
|
||||
}
|
||||
|
||||
+2
-2
@@ -156,12 +156,12 @@ public class GuzhiController {
|
||||
return mv;
|
||||
}
|
||||
|
||||
|
||||
//福州农商行人工录入
|
||||
public Object checkAction(@Valid GuzhiCheckModel model){
|
||||
RestfulModelAndView mv = new RestfulModelAndView();
|
||||
|
||||
mv.setModel(BasicModel.buildOKModel());
|
||||
String msg = "【房价网-人工审查地址】接收到新的人工审查地址信息,请求请及时处理!城市:" + model.getCity()
|
||||
String msg = "【房价网-人工审查地址】接收到新的人工审查地址信息,请求请及时处理!城市:" + model.getCity()+" 区域:"+model.getDistrict() +" 小区名称:"+(model.getName()!=null?model.getName():"")
|
||||
+ " 产证地址:" + model.getAddress();
|
||||
DingDingMessageUtil.sendTextMessage("065f4a05d5f87eb39a9f9e048eeefb80c1f7560446f8926441de80453c160c60", msg);
|
||||
return mv;
|
||||
|
||||
+1
-1
@@ -671,7 +671,7 @@ public class PropertyController {
|
||||
&& !"1007486".equals(userId) && !"1007533".equals(userId) && !"1007630".equals(userId)
|
||||
&& !"1007705".equals(userId) && !"1007739".equals(userId) && !"1007736".equals(userId)
|
||||
&& !"1007741".equals(userId) && !"1007742".equals(userId) && !"1007743".equals(userId)
|
||||
&& !"1007747".equals(userId)
|
||||
&& !"1007747".equals(userId) && !"1007757".equals(userId)
|
||||
&& model.getDetailLevel() != null && model.getDetailLevel() != 0) {// 临时权限控制
|
||||
mav.setModel(BasicModel.buildUnAuthorizedModel());
|
||||
return mav;
|
||||
|
||||
+10
@@ -284,6 +284,8 @@ public class PropertyEvaluationModel extends FangjiaApiModel {
|
||||
|
||||
private String user;
|
||||
|
||||
private Integer avgPrice;
|
||||
|
||||
|
||||
public void setDecorationAge(int decorationAge) {
|
||||
this.decorationAge = decorationAge;
|
||||
@@ -895,4 +897,12 @@ public class PropertyEvaluationModel extends FangjiaApiModel {
|
||||
public void setAtticSize(Double atticSize) {
|
||||
this.atticSize = atticSize;
|
||||
}
|
||||
|
||||
public Integer getAvgPrice() {
|
||||
return avgPrice;
|
||||
}
|
||||
|
||||
public void setAvgPrice(Integer avgPrice) {
|
||||
this.avgPrice = avgPrice;
|
||||
}
|
||||
}
|
||||
|
||||
+2
-2
@@ -30,12 +30,12 @@ public class MonthlyDistrictStatCalcBatch extends BaseBatch {
|
||||
void process(Object args) {
|
||||
List<String> cities = [];
|
||||
def allCity = CommonUtils.findAllCity()
|
||||
allCity = ["ËÕÖÝ"]
|
||||
|
||||
def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1)
|
||||
cities.addAll(allCity)
|
||||
def districts = []
|
||||
cities.each {def city->
|
||||
districts.addAll DbUtils.seaweed.find(new BasicDBObject("city", city).append("cat", "district").append("visible", 0).append("name", "ÎáÔù㳡"), keys).batchSize(2000).toList()
|
||||
districts.addAll DbUtils.seaweed.find(new BasicDBObject("city", city).append("cat", "district").append("visible", 0), keys).batchSize(2000).toList()
|
||||
println(city)
|
||||
}
|
||||
ExecutorService executorService = Executors.newFixedThreadPool(20)
|
||||
|
||||
+6
-4
@@ -54,6 +54,10 @@ class DealPriceMathBatch extends BaseBatch{
|
||||
|
||||
|
||||
def flag = true;
|
||||
|
||||
def startDate = DateUtil.getFirstDayOfMonth(new Date(),-1)
|
||||
def endDate = DateUtil.getFirstDayOfMonth(new Date(), 1)
|
||||
|
||||
cityMap.each{city,alias->
|
||||
println "开始加载【${city}】小区"
|
||||
if(city != "昆玉"){
|
||||
@@ -61,10 +65,8 @@ class DealPriceMathBatch extends BaseBatch{
|
||||
districtMap.putAll(detailMap)
|
||||
}
|
||||
}
|
||||
def startDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2020-01-01","yyyy-MM-dd"))
|
||||
// def startDate = DateUtil.getFirstDayOfMonth(new Date())
|
||||
// def endDate = DateUtil.getFirstDayOfMonth(new Date(), 1)
|
||||
def endDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2021-04-30","yyyy-MM-dd"))
|
||||
// def startDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2020-01-01","yyyy-MM-dd"))
|
||||
// def endDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2021-04-30","yyyy-MM-dd"))
|
||||
|
||||
while(flag){
|
||||
println "开始加载解析数据......"
|
||||
|
||||
+23
-11
@@ -211,23 +211,35 @@ class AccessLogBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
def read(){
|
||||
String fileName = "C:\\Users\\gaojuan\\Desktop\\localhost_access_log.2019-06-01.txt";
|
||||
String fileName = "D:\\Êý¾Ý\\logs\\performance.open.log";
|
||||
File file = new File(fileName);
|
||||
def rets = []
|
||||
file.eachLine{
|
||||
def index = 0
|
||||
def list =[]
|
||||
def slowCount = 0
|
||||
file.eachLine{it->
|
||||
|
||||
if(it.contains("/fangjiatong2/")){
|
||||
try{
|
||||
it = it.replace(it, URLDecoder.decode(URLDecoder.decode(it, "utf-8"), "UTF-8"))
|
||||
}catch(IllegalArgumentException e){}
|
||||
|
||||
if(it.contains("ws/czbank") && it.contains("entity:House")){
|
||||
// println it
|
||||
index++
|
||||
// try{
|
||||
def s1 = it.substring(it.indexOf("time:")+5, it.length())
|
||||
s1 = s1.substring(0, s1.indexOf("\t"))
|
||||
list << s1.toInteger()
|
||||
if(s1.isNumber() && s1.toInteger()>=4000){
|
||||
println it
|
||||
slowCount++
|
||||
}
|
||||
// }catch(IllegalArgumentException e){}
|
||||
}
|
||||
|
||||
if(!it.contains("äîϪËÄ´å")){
|
||||
rets << it
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
writeTxt(rets, "C:\\Users\\gaojuan\\Desktop\\fjt-2020-6-1.txt");
|
||||
println list.sum()/list.size()
|
||||
println index
|
||||
println "slowCount:"+slowCount
|
||||
// writeTxt(rets, "C:\\Users\\gaojuan\\Desktop\\fjt-2020-6-1.txt");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+19
-4
@@ -102,7 +102,7 @@ class DistrictInfoExportBatch extends BaseBatch{
|
||||
row << rMap.estate_company
|
||||
row << getBuildStructure(rMap.build_structure)//为空的,默认钢混
|
||||
row << rMap.loudong_count
|
||||
row << processAlias(rMap.alias)
|
||||
row << processAlias(rMap.alias, rMap.addr_alias)
|
||||
|
||||
def addr = fzAddrMap.get(rMap.get("_id").toString());
|
||||
if(addr){
|
||||
@@ -110,20 +110,32 @@ class DistrictInfoExportBatch extends BaseBatch{
|
||||
}else{
|
||||
row << rMap.address
|
||||
}
|
||||
//row << processAlias(rMap.addr_alias)
|
||||
result << row
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
def processAlias(alias){
|
||||
def processAlias(alias, def addr_alias){
|
||||
def newAlias = []
|
||||
if(alias){
|
||||
def newAlias = []
|
||||
for(def name : alias){
|
||||
if(name && !name.contains("(") && !name.contains("¡¢") && !name.contains("?") && !name.contains("£¨")){
|
||||
if(name && !name.contains("(") && !name.contains("¡¢") && !name.contains("?") && !name.contains("£¨") && !newAlias.contains(name)){
|
||||
newAlias << name
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(addr_alias){
|
||||
for(def name : addr_alias){
|
||||
if(name && !name.contains("(") && !name.contains("¡¢") && !name.contains("?") && !name.contains("£¨") && !newAlias.contains(name)){
|
||||
newAlias << name
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(newAlias){
|
||||
return newAlias.join(",");
|
||||
}
|
||||
return null;
|
||||
@@ -165,6 +177,9 @@ class DistrictInfoExportBatch extends BaseBatch{
|
||||
rets << "非钢混"
|
||||
}
|
||||
}
|
||||
if(rets.size()==0){
|
||||
rets << "¸Ö»ì"
|
||||
}
|
||||
return rets.join(",")
|
||||
}
|
||||
return "钢混";
|
||||
|
||||
+2
-2
@@ -76,8 +76,8 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
def createDatas(def city){
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", null);
|
||||
|
||||
def lastDate = DateUtils.getLastDayOfMonth(new Date(), -1)
|
||||
def startDate = DateUtils.getFirstDayOfMonth(lastDate);
|
||||
def lastDate = new Date()
|
||||
def startDate = DateUtils.getFirstDayOfMonth(lastDate, -1)
|
||||
|
||||
def result = []
|
||||
for(def rMap : districts){
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
package com.fangjia.script.fuzhou
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
|
||||
def count0 = 0
|
||||
def count1 = 0
|
||||
def count2 = 0
|
||||
def count3 = 0
|
||||
def count4 = 0
|
||||
def count5 = 0
|
||||
def count6 = 0
|
||||
def count7 = 0
|
||||
def count8 = 0
|
||||
def total=4943
|
||||
for(def d : datas){
|
||||
def list = []
|
||||
if(d['G']){
|
||||
list << d['G']
|
||||
}
|
||||
if(d['H']){
|
||||
list << d['H']
|
||||
}
|
||||
if(d['I']){
|
||||
list << d['I']
|
||||
}
|
||||
if(d['J']){
|
||||
list << d['J']
|
||||
}
|
||||
if(d['K']){
|
||||
list << d['K']
|
||||
}
|
||||
if(d['L']){
|
||||
list << d['L']
|
||||
}
|
||||
if(d['M']){
|
||||
list << d['M']
|
||||
}
|
||||
if(d['N']){
|
||||
list << d['N']
|
||||
}
|
||||
if(d['O']){
|
||||
list << d['O']
|
||||
}
|
||||
if(list.size()==0){
|
||||
count0++
|
||||
}
|
||||
if(list.size()==1){
|
||||
count1++
|
||||
}
|
||||
if(list.size()==2){
|
||||
count2++
|
||||
}
|
||||
if(list.size()==3){
|
||||
count3++
|
||||
}
|
||||
if(list.size()==4){
|
||||
count4++
|
||||
}
|
||||
if(list.size()==5){
|
||||
count5++
|
||||
}
|
||||
|
||||
if(list.size()==6){
|
||||
count6++
|
||||
}
|
||||
if(list.size()==7){
|
||||
count7++
|
||||
}
|
||||
if(list.size()==8){
|
||||
count8++
|
||||
}
|
||||
}
|
||||
|
||||
println("count0:"+ (DataHandleUtils.getDoubleRound(2, count0*100.0/total)))
|
||||
println("count1:"+ (DataHandleUtils.getDoubleRound(2, count1*100.0/total)))
|
||||
println("count2:"+ (DataHandleUtils.getDoubleRound(2, count2*100.0/total)))
|
||||
println("count3:"+ (DataHandleUtils.getDoubleRound(2, count3*100.0/total)))
|
||||
println("count4:"+ (DataHandleUtils.getDoubleRound(2, count4*100.0/total)))
|
||||
println("count5:"+ (DataHandleUtils.getDoubleRound(2, count5*100.0/total)))
|
||||
println("count6:"+ (DataHandleUtils.getDoubleRound(2, count6*100.0/total)))
|
||||
println("count7:"+ (DataHandleUtils.getDoubleRound(2, count7*100.0/total)))
|
||||
println("count8:"+ (DataHandleUtils.getDoubleRound(2, count8*100.0/total)))
|
||||
|
||||
|
||||
|
||||
////def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region", "name"]);
|
||||
//def total = 8018
|
||||
//def names = ["诸葛找房":"zhugezhaofang_esf_price","安居客":"anjuke_esf_price","搜房":"soufang_esf_price","链家":"lianjia_esf_price","城市房产":"cityhouse_esf_price","房探":"fangtan_esf_price","房多多":"fangdoudou_esf_price","麦田":"maitian_esf_price", "58同城":"58tc_esf_price"]
|
||||
//
|
||||
//def rets = []
|
||||
//def sDate = 202011
|
||||
//rets << ["时间","诸葛找房","安居客","搜房","链家","城市房产","房探","房多多","麦田", "58同城"]
|
||||
//def ret = []
|
||||
//ret << sDate
|
||||
//names.each{k,v->
|
||||
// def crawlCount = crawlCount(k, sDate)
|
||||
// def count = count(v, sDate)
|
||||
// println k +" "+DataHandleUtils.getDoubleRound(2, count*100.0/total)+"("+crawlCount+")"
|
||||
// ret << DataHandleUtils.getDoubleRound(2,count*100.0/total)+"("+crawlCount+")"
|
||||
//}
|
||||
//rets.add(ret)
|
||||
//ExcelUtils.write("d:\\数据\\第三方均价覆盖率1.xlsx",rets,"数据")
|
||||
//
|
||||
//def crawlCount(k, sDate){
|
||||
// def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
// def q = new BasicDBObject()
|
||||
// q.put("city","福州")
|
||||
// q.put("s_date", sDate)
|
||||
// q.put("source",k)
|
||||
// return thirdPartyColl.count(q)
|
||||
//}
|
||||
//
|
||||
//
|
||||
//def count(name, sDate){
|
||||
// DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
// def q = new BasicDBObject()
|
||||
// q.put("city","福州")
|
||||
// q.put("s_date", sDate)
|
||||
// q.put(name, new BasicDBObject('$gt', 0));
|
||||
// return AVERAGE_STATISTICS.count(q)
|
||||
//}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
+5
-5
@@ -18,7 +18,7 @@ import org.apache.commons.codec.digest.DigestUtils
|
||||
*/
|
||||
class ThirdPriceExportBatch extends BaseBatch{
|
||||
|
||||
def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM"));
|
||||
def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
@@ -156,12 +156,12 @@ class ThirdPriceExportBatch extends BaseBatch{
|
||||
if(!retMap.cityhouse_esf_price && d.cityhouse_esf_price){
|
||||
retMap.cityhouse_esf_price = d.cityhouse_esf_price
|
||||
}
|
||||
if(!retMap.fangtan_esf_price && d.fangtan_esf_price){
|
||||
retMap.fangtan_esf_price = d.fangtan_esf_price
|
||||
if(!retMap.fangtan_esf_price && d.jiwu_esf_price){//使用吉屋替换房探 待抓取
|
||||
retMap.fangtan_esf_price = d.jiwu_esf_price
|
||||
}
|
||||
|
||||
if(!retMap.fangdoudou_esf_price && d.fangdoudou_esf_price){
|
||||
retMap.fangdoudou_esf_price = d.fangdoudou_esf_price
|
||||
if(!retMap.fangdoudou_esf_price && d.get("365house_esf_price")){//使用淘房365替换房多多
|
||||
retMap.fangdoudou_esf_price = d.get("365house_esf_price")
|
||||
}
|
||||
|
||||
if(!retMap.maitian_esf_price && d.maitian_esf_price){
|
||||
|
||||
+37
-27
@@ -3,6 +3,9 @@ package com.fangjia.script.gj
|
||||
import com.fangjia.core.ProvinceData
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.httpclient.UserAgents
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.gj.paimai.GaoDeUtils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.JacksonUtil
|
||||
@@ -30,39 +33,40 @@ import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
import java.util.zip.GZIPInputStream
|
||||
|
||||
class GaoDeMapCrawl {
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class GaoDeMapCrawl extends BaseBatch{
|
||||
|
||||
DBCollection BOUNDS_COLL = MongoUtils.getCollectionByName("fangjia_base", "city_bounds_box", "192.168.0.235", 27018)
|
||||
static def targetUrl = "http://192.168.0.206:9200/amap_poi/_doc"
|
||||
def targetUrl = "http://192.168.0.206:9200/amap_poi/_doc"
|
||||
|
||||
public static final def dbUrl = "mongodb://192.168.0.235:27018";
|
||||
final def dbUrl = "mongodb://192.168.0.235:27018";
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1)
|
||||
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1)
|
||||
final AtomicInteger count = new AtomicInteger()
|
||||
def updateLists = []
|
||||
def updateErrorLists = []
|
||||
def flagLists = []
|
||||
def index = 0
|
||||
def prefix = "district_"
|
||||
|
||||
// def categorys = ["141202", "141203"]//中学,小学
|
||||
def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//住宅相关
|
||||
|
||||
// def prefix = "district_"
|
||||
// def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//住宅相关
|
||||
|
||||
// def categorys = ["190301"]//道路相关
|
||||
|
||||
def prefix = "village_"
|
||||
def categorys = ["190108","190109"]//村庄
|
||||
def totalCount = new AtomicLong()
|
||||
|
||||
static main(args) {
|
||||
def task = new GaoDeMapCrawl();
|
||||
|
||||
// def citys = CommonUtils.findAllCity()
|
||||
// def i =1
|
||||
// for(def city : citys){
|
||||
// i++
|
||||
// println i+" "+ city
|
||||
// }
|
||||
|
||||
task.run()
|
||||
// task.allRun();
|
||||
runBatch(new GaoDeMapCrawl(), args)
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
run()
|
||||
}
|
||||
def allRun(){
|
||||
def cityLations = null;
|
||||
def stop = false;
|
||||
@@ -110,14 +114,20 @@ class GaoDeMapCrawl {
|
||||
|
||||
|
||||
def run(){//,"西安","重庆"
|
||||
// def citys = [ "临沂","泰安","菏泽","东营","德州","济宁","日照","聊城","滨州","枣庄","莱芜","济南","青岛","威海","烟台","潍坊","淄博"]
|
||||
def citys = ["重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
|
||||
// def citys = CommonUtils.findAllCity()
|
||||
def totalCount = 0;
|
||||
for(def city : citys){
|
||||
def cityLations = findAllLation(city);
|
||||
println "开始处理["+city +"]:"+cityLations.size()
|
||||
gaoDeCitySearch(cityLations);
|
||||
def stop = false
|
||||
while(stop == false){
|
||||
def cityLations = findAllLation(city);
|
||||
if(!cityLations){
|
||||
stop = true;
|
||||
}else{
|
||||
println "开始处理["+city +"]:"+cityLations.size()
|
||||
gaoDeCitySearch(cityLations);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -129,7 +139,7 @@ class GaoDeMapCrawl {
|
||||
q.put(prefix+"status", new BasicDBObject(QueryOperators.EXISTS, false))
|
||||
// q.put("status", "失败")
|
||||
// q.put("traffic_status", new BasicDBObject(QueryOperators.EXISTS, false))
|
||||
def results = BOUNDS_COLL.find(q).batchSize(2000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def results = BOUNDS_COLL.find(q).limit(2000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
return results;
|
||||
}
|
||||
|
||||
@@ -142,7 +152,7 @@ class GaoDeMapCrawl {
|
||||
// void run() {
|
||||
// try{
|
||||
// totalCount.addAndGet(1)
|
||||
searching(location)
|
||||
searching(i, location)
|
||||
// }catch(Exception e){
|
||||
// e.printStackTrace()
|
||||
// } finally {
|
||||
@@ -169,7 +179,7 @@ class GaoDeMapCrawl {
|
||||
}
|
||||
}
|
||||
|
||||
def searching(location){
|
||||
def searching(i, location){
|
||||
// def bound_gd = location.bound_gd
|
||||
// def lats = [bound_gd[1], bound_gd[3]]
|
||||
// def lngs = [bound_gd[0], bound_gd[2]]
|
||||
@@ -177,7 +187,7 @@ class GaoDeMapCrawl {
|
||||
// lngs.sort()
|
||||
// println index
|
||||
// if(lngs[0]<=121.418153 && lngs[1]>=121.418153 && lats[0]<=31.218824 && lats[1]>=31.218824){
|
||||
search(location)
|
||||
search(i, location)
|
||||
// }
|
||||
|
||||
if(updateLists.size() > 0 && updateLists.size()>1000){
|
||||
@@ -196,7 +206,7 @@ class GaoDeMapCrawl {
|
||||
}
|
||||
|
||||
|
||||
def search(location){
|
||||
def search(i, location){
|
||||
def bound_gd = location.bound_gd
|
||||
def lats = [bound_gd[1], bound_gd[3]]
|
||||
def lngs = [bound_gd[0], bound_gd[2]]
|
||||
@@ -230,7 +240,7 @@ class GaoDeMapCrawl {
|
||||
updateErrorLists.addAll(location)
|
||||
}
|
||||
// update(location);
|
||||
println "[ "+totalCount.get()+" ]"+location._id+" "+location.city+" "+location.bound_gd+" flag: "+m.status
|
||||
println "[ "+i+" ]"+location._id+" "+location.city+" "+location.bound_gd+" flag: "+m.status
|
||||
}
|
||||
|
||||
def processData(rets){
|
||||
|
||||
@@ -43,6 +43,24 @@ class OpenApiUtils {
|
||||
}
|
||||
|
||||
|
||||
static def matchDetailDistrict(def city, def address, def suggestDistrict, def preciseLevelflag){
|
||||
address = address.replace("£¨","(").replace("£©",")").replace(" ","")
|
||||
def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&suggestDistrict="+suggestDistrict+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1"
|
||||
if(preciseLevelflag){
|
||||
url = url +"&preciseLevel=strict"
|
||||
}
|
||||
String content = new HttpRequest().get(url, "utf-8");
|
||||
if(content){
|
||||
def doc = new JsonSlurper().parseText(content)
|
||||
if(doc && doc.code ==200 && doc.result){
|
||||
def dto = doc.result
|
||||
return dto
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
static def matchDistrict2(def city, def address, def preciseLevelflag){
|
||||
address = address.replace("£¨","(").replace("£©",")").replace(" ","").replace("#","")
|
||||
def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1"
|
||||
|
||||
+2
-2
@@ -47,7 +47,7 @@ class YouDaDataParser {
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
|
||||
|
||||
static def statDate = "2021-03-01"
|
||||
static def statDate = "2021-04-01"
|
||||
static main(args) {
|
||||
def task = new YouDaDataParser();
|
||||
// task.newHouseparse();//友达一手房存入mongo
|
||||
@@ -285,7 +285,7 @@ class YouDaDataParser {
|
||||
|
||||
|
||||
def esfParser(){
|
||||
def files = ["D:\\数据\\友达\\2021年4月\\二手房\\查询1.DBF"]
|
||||
def files = ["D:\\数据\\友达\\2021年5月\\二手房\\查询1.DBF"]
|
||||
def datas = []
|
||||
for(def fileName : files){
|
||||
def rows = readDBF(fileName);
|
||||
|
||||
+77
@@ -0,0 +1,77 @@
|
||||
package com.fangjia.script.gj.ccb
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
class CcbCityPriceStat {
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
static main(args) {
|
||||
def task = new CcbCityPriceStat()
|
||||
task.run()
|
||||
}
|
||||
def run(){
|
||||
def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
|
||||
def rets = []
|
||||
for(def city : citys){
|
||||
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "alias"])
|
||||
def priceMap = CommonDatautils.queryDistrictPrice(city, DateUtils.parseFromTightYYYYMMDD("2021-05-30"))
|
||||
def cityRets = []
|
||||
for(def dist : districts){
|
||||
def tradeAvgPrice = findTrades(dist, "2021-02-01", "2021-06-01")
|
||||
def weekendPrice = priceMap.get(city +"##"+dist.region+"##"+dist.name)
|
||||
if(tradeAvgPrice && weekendPrice){
|
||||
def row = [city, dist.region, dist.name, weekendPrice, tradeAvgPrice.toInteger()]
|
||||
cityRets << row
|
||||
println row.toString()
|
||||
}
|
||||
}
|
||||
print city +" "+ cityRets.size()
|
||||
rets.addAll(cityRets)
|
||||
}
|
||||
|
||||
println "总量:"+rets.size()
|
||||
ExcelUtils.write("d:\\数据\\92个城市价格数据.xlsx", rets, "所有")
|
||||
}
|
||||
|
||||
|
||||
def findTrades(dist, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", dist.city)
|
||||
q.put("region", dist.region)
|
||||
|
||||
if(dist.alias){
|
||||
if(!dist.alias.contains(dist.name)){
|
||||
dist.alias << dist.name
|
||||
}
|
||||
q.put("district_name", new BasicDBObject('$in', dist.alias))
|
||||
}else{
|
||||
q.put("district_name", dist.name)
|
||||
}
|
||||
|
||||
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate)))
|
||||
def dbcorsor = DEAL_PRICE_COLL.find(q)
|
||||
def avg_price = 0
|
||||
def total_price = 0
|
||||
def count = 0
|
||||
|
||||
for(def d : dbcorsor){
|
||||
if(d.avg_price){
|
||||
total_price = total_price + d.avg_price
|
||||
count++
|
||||
}
|
||||
}
|
||||
if(count > 3){
|
||||
avg_price = total_price/count
|
||||
}
|
||||
return avg_price
|
||||
}
|
||||
|
||||
}
|
||||
+322
@@ -0,0 +1,322 @@
|
||||
package com.fangjia.script.gj.chongqingblank
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
import com.fangjia.script.gj.*
|
||||
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class DataMatch {
|
||||
|
||||
static final DBCollection FANGJIA_OFFICIAL_AREA_MAP = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
|
||||
static main(args) {
|
||||
def task = new DataMatch()
|
||||
// task.run1()
|
||||
task.run2()
|
||||
// task.run3()
|
||||
// println task.findAddress("昌州街道棠香路")
|
||||
|
||||
// task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx")
|
||||
}
|
||||
|
||||
def run1(){
|
||||
def fileName = "D:\\数据\\重庆农商行\\估值2-V5.xlsx"
|
||||
//matchRegionValue(fileName);
|
||||
matchDistrict(fileName)
|
||||
// ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
// if(d['O']=="新增"){
|
||||
//
|
||||
// d['L'] = d['P']
|
||||
// d['M'] = d['Q']
|
||||
// d['N'] = d['R']
|
||||
// }
|
||||
// },0)
|
||||
|
||||
}
|
||||
|
||||
def parseAddress(def fileName){
|
||||
ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
|
||||
},0)
|
||||
}
|
||||
|
||||
def matchDistrict(fileName){
|
||||
ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
def region = d['Q']
|
||||
def city = d['P']
|
||||
// if(!d['I'] && !d['J']){
|
||||
// city = d['B']
|
||||
// region = d['C']
|
||||
// }
|
||||
if(d['R'] && !d['G']){
|
||||
def results = null
|
||||
// if(d['E'] && isStreet(d['E'])){//加上效果并没有想象中的好
|
||||
// results = OpenApiClient.addressMatch(city, d['E'], "property", false, region, d['D'])
|
||||
// }
|
||||
if((results && results.precise<0) || !results){
|
||||
results = OpenApiClient.addressMatch(city, d['R'], "property", false, region, null)
|
||||
}
|
||||
// if((d['E'] && results && results.precise<0) || !results){
|
||||
// results = OpenApiClient.addressMatch(city, d['E'], "property", false, region, d['D'])
|
||||
// }
|
||||
if(results && results.precise>=0){
|
||||
d['K'] = results.city
|
||||
d['L'] = results.district
|
||||
d['M'] = results.name
|
||||
d['N'] = results.address
|
||||
}
|
||||
}
|
||||
},0);
|
||||
}
|
||||
|
||||
def run2(){
|
||||
def fileName = "D:\\数据\\重庆农商行\\估值2-V2.xlsx"
|
||||
// matchRegionValue(fileName);
|
||||
matchDistrict2(fileName)
|
||||
}
|
||||
|
||||
|
||||
def matchDistrict2(fileName){
|
||||
ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
def region = d['I']
|
||||
def city = d['J']
|
||||
if(!d['I']){
|
||||
city = d['J']
|
||||
}
|
||||
if(city && d['E'] && !d['O'] ){
|
||||
def addressResult = OpenApiClient.addressStandard(city, d['E'])
|
||||
if(addressResult){
|
||||
if(addressResult.street && addressResult.streetNumber){
|
||||
d['H'] = addressResult.street+addressResult.streetNumber
|
||||
}
|
||||
}
|
||||
def results = null
|
||||
if(d['H']){
|
||||
results = OpenApiClient.addressMatch(city, d['H'], "property", false, region, null)
|
||||
}
|
||||
|
||||
if((d['E'] && results && results.precise<0) || !results){
|
||||
results = OpenApiClient.addressMatch(city, d['E'], "property", false, region, null)
|
||||
}
|
||||
|
||||
if(results && results.precise>=0){
|
||||
d['K'] = results.city
|
||||
d['L'] = results.district
|
||||
d['M'] = results.name
|
||||
d['N'] = results.address
|
||||
}
|
||||
}
|
||||
},0);
|
||||
}
|
||||
|
||||
def run3(){
|
||||
def fileName = "D:\\数据\\重庆农商行\\估值3.xlsx"
|
||||
// matchDistrict3(fileName)
|
||||
ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
if(d['I']){
|
||||
d['I'] = d['I'].toString().replace("省","").replace("市","")
|
||||
}
|
||||
if(d['J'] && d['I']){
|
||||
d['J'] = RegionData.formatRegion(d['I'], d['J'])
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
|
||||
def matchDistrict3(fileName){
|
||||
ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
def address = d['B']
|
||||
if(address) {
|
||||
def arrs = address.split("->")
|
||||
def city = null
|
||||
def region = null
|
||||
def addr = null
|
||||
if (arrs && arrs.size() == 4) {
|
||||
city = arrs[0]
|
||||
region = arrs[2]
|
||||
addr = arrs[3]
|
||||
d['I'] = city
|
||||
d['J'] = region
|
||||
} else if (arrs && arrs.size() == 2) {
|
||||
def parseCity = CommonUtils.parseCityInAddress(arrs[1])
|
||||
city = parseCity
|
||||
d['I'] = parseCity
|
||||
addr = arrs[1]
|
||||
}
|
||||
if (addr && !d['O']) {
|
||||
def addressResult = OpenApiClient.addressStandard(city, addr)
|
||||
if (addressResult) {
|
||||
if (!region) {
|
||||
region = addressResult.district
|
||||
d['J'] = region
|
||||
}
|
||||
if (addressResult.street && addressResult.streetNumber) {
|
||||
d['H'] = addressResult.street + addressResult.streetNumber
|
||||
}
|
||||
}
|
||||
def results = null
|
||||
def name = null
|
||||
if (d['C']) {
|
||||
name = d['C'].toString().replace(".", "").replace("(", "").replace(")", "").replace("。", "")
|
||||
.replace("·", "").replace("-", "").replace("`", "").replace("?", "").replace("‘", "")
|
||||
.replace("1期", "一期").replace("2期", "二期").replace("3期", "三期")
|
||||
.replace("4期", "四期").replace("5期", "五期").replace("12区", "十二区").replace("1区", "一区").replace("2区", "二区").replace("3区", "三区")
|
||||
.replace("4区", "四区").replace("5区", "五区").replace("a区", "A区")
|
||||
.replace("B区", "B区").replace("c区", "C区").replace("d区", "D区")
|
||||
}
|
||||
if (name) {
|
||||
results = OpenApiClient.addressMatch(city, name, "property", false, region, null)
|
||||
}
|
||||
|
||||
if ((addr && results && results.precise < 0) || !results) {
|
||||
results = OpenApiClient.addressMatch(city, addr, "property", false, region, name)
|
||||
}
|
||||
if (results && results.precise >= 0) {
|
||||
d['K'] = results.city
|
||||
d['L'] = results.district
|
||||
d['M'] = results.name
|
||||
d['N'] = results.address
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
},0);
|
||||
}
|
||||
|
||||
|
||||
|
||||
def matchRegionValue(fileName){
|
||||
ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
def code = d['C']
|
||||
if(code && code.toString().isNumber()){
|
||||
if(code.toInteger().toString()=="500226"){
|
||||
d['I'] = "荣昌"
|
||||
d['J'] = "重庆"
|
||||
}else if(code.toInteger().toString()=="500223"){
|
||||
d['I'] = "潼南"
|
||||
d['J'] = "重庆"
|
||||
}else{
|
||||
def data = findRegionCodeValue(code.toInteger().toString());
|
||||
if(data){
|
||||
d['I'] = data.fangjia_district
|
||||
d['J'] = data.fangjia_city
|
||||
}else{
|
||||
data = findCityCodeValue(code.toInteger().toString())
|
||||
if(data){
|
||||
d['J'] = data.fangjia_city
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
},0)
|
||||
}
|
||||
|
||||
def findCityCodeValue(value){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city_code", value)
|
||||
def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q)
|
||||
def count = dbcursor.count()
|
||||
if(count >1){
|
||||
def data = null
|
||||
for(def db : dbcursor){
|
||||
data = db
|
||||
break
|
||||
}
|
||||
return data;
|
||||
}
|
||||
}
|
||||
|
||||
def findRegionCodeValue(value){
|
||||
def q = new BasicDBObject()
|
||||
q.put("district_code", value)
|
||||
def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q)
|
||||
def count = dbcursor.count()
|
||||
if(count == 1){
|
||||
def data = null
|
||||
for(def db : dbcursor){
|
||||
data = db
|
||||
break
|
||||
}
|
||||
return data;
|
||||
}
|
||||
}
|
||||
|
||||
def findAddress(String address) {
|
||||
//Pattern ROAD_PATTERN = Pattern.compile('.*[路|大道|巷|胡同|街]\\d+号');
|
||||
Pattern ROAD_PATTERN = Pattern.compile('.[里,路,道,街,巷,路段,胡同].?\\d+.?\\d+[号,弄]|.[里,路,道,街,巷,路段,胡同].?\\d+.?[号,弄]')
|
||||
Matcher m = ROAD_PATTERN.matcher(address)
|
||||
String newAddress
|
||||
if (m.find()) {
|
||||
newAddress = m.group()
|
||||
}
|
||||
return newAddress
|
||||
}
|
||||
|
||||
def run2MatchV2(fileName){
|
||||
ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
def region = d['I']
|
||||
def city = d['J']
|
||||
if(!d['I'] && !d['J']){
|
||||
city = d['B']
|
||||
region = d['C']
|
||||
}
|
||||
if(d['D']) {
|
||||
def name = d['D'].toString().replace(".", "").replace("(", "").replace(")", "").replace("。", "")
|
||||
.replace("·", "").replace("-", "").replace("`", "").replace("?", "").replace("‘", "")
|
||||
.replace("1期", "一期").replace("2期", "二期").replace("3期", "三期")
|
||||
.replace("4期", "四期").replace("5期", "五期").replace("12区", "十二区").replace("1区", "一区").replace("2区", "二区").replace("3区", "三区")
|
||||
.replace("4区", "四区").replace("5区", "五区").replace("a区", "A区")
|
||||
.replace("B区", "B区").replace("c区", "C区").replace("d区", "D区")
|
||||
|
||||
def result = OpenApiUtils.matchDetailDistrict(city, name, region, true)
|
||||
if (!result && d['E']) {
|
||||
if (findAddress(d['E'])) {
|
||||
result = OpenApiUtils.matchDetailDistrict(city, d['E'], region, true)
|
||||
}
|
||||
}
|
||||
if(result){
|
||||
def matchResult = result.matchResult
|
||||
if (result.credit > 0.5 && result.matchResult && !result.matchResult.match) {
|
||||
matchResult = amendMatch(result, d['E'])
|
||||
}
|
||||
if (matchResult.match) {
|
||||
d['AQ'] = result.city
|
||||
d['AR'] = result.district
|
||||
d['AS'] = result.name
|
||||
d['AT'] = result.address
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
},0)
|
||||
}
|
||||
|
||||
def amendMatch(result, address){
|
||||
def matchResult = result.matchResult
|
||||
if(result.credit > 0.65 && matchResult.distance < 2500){
|
||||
matchResult.match = true
|
||||
}else if(result.credit>0.5 && address && result.address && findAddress(result.address)){
|
||||
if(result.address.contains(address) || address.contains(result.address)){
|
||||
matchResult.match = true
|
||||
}
|
||||
}else if(address && address.contains(result.district) && address.contains(result.name)){
|
||||
matchResult.match = true
|
||||
}
|
||||
return matchResult
|
||||
}
|
||||
}
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
package com.fangjia.script.gj.chongqingblank
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import net.sf.json.JSONObject
|
||||
|
||||
class EvaluationPrice {
|
||||
|
||||
static main(args) {
|
||||
def task = new EvaluationPrice()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\估值3-V3.xlsx",{ d->
|
||||
if(d['K'] && d['M']){
|
||||
def priceMap = getEvaluationPrice(d['J'], d['K'], d['L'], d['D'].toString())
|
||||
if(priceMap){
|
||||
d['E'] = priceMap.avgPrice
|
||||
d['F'] = priceMap.totalPrice
|
||||
}
|
||||
}
|
||||
},0);
|
||||
}
|
||||
|
||||
|
||||
def getEvaluationPrice(city, region, name, area){
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("city", city);
|
||||
params.put("district", region);
|
||||
params.put("name", name);
|
||||
params.put("size", area);
|
||||
params.put("__refresh_cache","1")
|
||||
|
||||
JSONObject result = OpenApiClient.evaluate(params);
|
||||
if (result.getInt("code") == 200) {
|
||||
if (result.containsKey("result")) {
|
||||
return result.getJSONObject("result")
|
||||
}
|
||||
} else if (result.getInt("code") == 500) {
|
||||
println("小区估价失败:${params.city} ${params.name} ${params.size} ${result}")
|
||||
}
|
||||
}
|
||||
}
|
||||
+629
@@ -0,0 +1,629 @@
|
||||
package com.fangjia.script.gj.chongqingblank
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.datastore.driver.DSCollection
|
||||
import com.fangjia.datastore.driver.DataStore
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.gj.district.NameProcess
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeMapApiData
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBObject
|
||||
|
||||
class TestMatchResult {
|
||||
|
||||
private static final DSCollection SEAWEED = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed")
|
||||
|
||||
static main(args) {
|
||||
def task = new TestMatchResult()
|
||||
// task.run1("D:\\数据\\重庆农商行\\估值1.xlsx")
|
||||
task.run2("D:\\数据\\重庆农商行\\估值2-V2.xlsx")
|
||||
// task.run3("D:\\数据\\重庆农商行\\估值3.xlsx")
|
||||
|
||||
// task.nearDistRun("D:\\数据\\重庆农商行\\估值2-V5.xlsx")
|
||||
// task.run2V1("D:\\数据\\重庆农商行\\估值2-V2.xlsx")
|
||||
|
||||
// task.similarScore("D:\\数据\\重庆农商行\\估值2-V5.xlsx")
|
||||
|
||||
// task.mapRun("D:\\数据\\重庆农商行\\估值2-V4-map.xlsx");
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
def run1(def fileName){
|
||||
ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
def region = d['I']
|
||||
def city = d['J']
|
||||
if (!d['I'] && !d['J']) {
|
||||
city = d['B']
|
||||
region = d['C']
|
||||
}
|
||||
def newName = null
|
||||
if(d['D']=="大川菲尔小城"){
|
||||
if (d['D']) {
|
||||
newName = d['D'].toString().replace(".", "").replace("(", "").replace(")", "").replace("。", "")
|
||||
.replace("·", "").replace("-", "").replace("`", "").replace("?", "").replace("‘", "")
|
||||
.replace("1期", "一期").replace("2期", "二期").replace("3期", "三期")
|
||||
.replace("4期", "四期").replace("5期", "五期").replace("12区", "十二区").replace("1区", "一区").replace("2区", "二区").replace("3区", "三区")
|
||||
.replace("4区", "四区").replace("5区", "五区").replace("a区", "A区")
|
||||
.replace("B区", "B区").replace("c区", "C区").replace("d区", "D区")
|
||||
}
|
||||
if (!region || (region && region == d['L'])) {
|
||||
if (d['D'] && !d['O'] && d['M']) {
|
||||
if (d['M'] == newName) {
|
||||
d['O'] = "相同"
|
||||
}
|
||||
if (!d['O']) {
|
||||
def dbo = CommonDatautils.findDistrictByAlias(d['K'], d['L'], d['D'], "district")
|
||||
if (dbo) {
|
||||
d['O'] = "别名匹配"
|
||||
}
|
||||
}
|
||||
// if (!d['O'] && d['M'] && d['E']) {
|
||||
// if (d['E'].toString().contains(d['M'])) {
|
||||
// d['O'] = "名称in地址"
|
||||
// }
|
||||
// }
|
||||
|
||||
if (!d['O']) {
|
||||
def py1 = com.fangjia.core.util.DataUtil.hanyuQuanpin(newName, " ")
|
||||
def py2 = com.fangjia.core.util.DataUtil.hanyuQuanpin(d['M'], " ")
|
||||
if (py1 == py2) {
|
||||
d['O'] = "拼音相同"
|
||||
}
|
||||
}
|
||||
|
||||
if (!d['O'] && d['D'] && d['M'] && newName) {
|
||||
def word = null
|
||||
if (newName.toString().length() >= d['M'].toString().length()) {
|
||||
word = newName.toString().replace(d['M'], "")
|
||||
} else if (newName.toString().length() < d['M'].toString().length()) {
|
||||
word = d['M'].toString().replace(newName, "")
|
||||
}
|
||||
if (word) {
|
||||
def words = ["重庆", "小区", "(高层)", "花园", "恒都", "(洋房)", "社区"]
|
||||
for (def w : words) {
|
||||
if (word == w) {
|
||||
d['O'] = "去部分词后相同"
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if (!d['O'] && !word.endsWith("区") && !word.endsWith("期") && !word.endsWith("团") && !word.endsWith("东苑") && !word.endsWith("西苑") && !word.endsWith("南苑") && !word.endsWith("北苑")) {
|
||||
if (d['N'] && d['N'].toString().endsWith("号") && d['E'] && d['E'].toString().contains(d['N'])) {
|
||||
|
||||
def score = CosineSimilarAlgorithm.getSimilarity(d['M'], newName);
|
||||
if (score >= 0.65) {
|
||||
println newName + " " + d['M'] + " " + score
|
||||
d['O'] = "地址相同"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!d['O'] && newName && !newName.contains("期") && !newName.endsWith("组团") && (!newName.endsWith("区") || newName.endsWith("小区"))) {
|
||||
|
||||
def gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, d['E'], "住宅", "高德")
|
||||
if (!gaodeRest) {
|
||||
gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, d['E'], "住宅", "百度")
|
||||
}
|
||||
if (gaodeRest && gaodeRest.name) {
|
||||
def mapName = gaodeRest.name.replace(".", "").replace("(", "").replace(")", "").replace("·", "").replace("`", "").replace("?", "")
|
||||
if (mapName == d['M'] && gaodeRest.score >= 0.7) {
|
||||
d['O'] = gaodeRest.source + "匹配"
|
||||
println newName + " " + d['M'] + " " + gaodeRest.score
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if (d['D'] && !d['O']) {
|
||||
def gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, d['E'], "住宅", "高德")
|
||||
def mapName = null
|
||||
if (gaodeRest && gaodeRest.name) {
|
||||
mapName = gaodeRest.name.replace(".", "").replace("(", "").replace(")", "").replace("·", "").replace("`", "").replace("?", "")
|
||||
}
|
||||
if (!gaodeRest || (gaodeRest && mapName != newName)) {
|
||||
gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, d['E'], "住宅", "百度")
|
||||
if (gaodeRest && gaodeRest.name) {
|
||||
mapName = gaodeRest.name.replace(".", "").replace("(", "").replace(")", "").replace("·", "").replace("`", "").replace("?", "")
|
||||
}
|
||||
}
|
||||
if (gaodeRest && mapName == newName) {
|
||||
d['P'] = gaodeRest.district
|
||||
d['Q'] = mapName
|
||||
d['R'] = gaodeRest.address
|
||||
d['S'] = gaodeRest.lng
|
||||
d['T'] = gaodeRest.lat
|
||||
d['U'] = gaodeRest.type
|
||||
d['v'] = gaodeRest.source
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
},0);
|
||||
}
|
||||
|
||||
def similarScore(def fileName){
|
||||
ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
if(d['D'] && d['M']){
|
||||
def score = CosineSimilarAlgorithm.getSimilarity(d['D'], d['M']);
|
||||
d['Q'] = score
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
|
||||
def run2(def fileName){
|
||||
ExcelUtils.multiThreadingModify(fileName,{ d->
|
||||
def region = d['I']
|
||||
def city = d['J']
|
||||
if(!d['I']){
|
||||
city = d['J']
|
||||
}
|
||||
|
||||
// if(d['E']=="巴南区花溪镇花溪新街88号2单元3-4#") {
|
||||
// println "ss"
|
||||
|
||||
|
||||
if (!region || (region && region == d['L'])) {
|
||||
if (d['N'] && !d['O'] && d['E']) {
|
||||
if (d['N'].toString().endsWith("号") && d['E'] && d['E'].toString().contains(d['N'])) {
|
||||
d['O'] = "地址匹配"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!d['O'] && d['M'] && d['E']) {
|
||||
def matchName = d['M']
|
||||
if(matchName.toString().endsWith("小区")){
|
||||
matchName = matchName.toString().replace("小区","")
|
||||
}
|
||||
if(matchName.toString().startsWith(d['L'])){
|
||||
matchName = matchName.toString().replace(d['L'],"")
|
||||
}
|
||||
def oldName= NameProcess.nameStandard(city,region, d['E'])
|
||||
if (oldName.toString().contains(matchName)) {
|
||||
d['O'] = "名称in地址"
|
||||
}
|
||||
}
|
||||
|
||||
if (d['E'] && !d['O'] ) {
|
||||
def gaodeRest = null
|
||||
|
||||
if (!gaodeRest && d['E']) {
|
||||
gaodeRest = FangjiaClientUtils.getMapData(city, region, d['E'], null, "住宅", "高德")
|
||||
}
|
||||
if(!gaodeRest && d['H']){
|
||||
gaodeRest = FangjiaClientUtils.getMapData(city, region, d['H'], null, "住宅", "高德")
|
||||
}
|
||||
if (!gaodeRest && d['E']) {
|
||||
gaodeRest = FangjiaClientUtils.getMapData(city, region, d['E'], null, "住宅", "百度")
|
||||
}
|
||||
if (!gaodeRest && d['H']) {
|
||||
gaodeRest = FangjiaClientUtils.getMapData(city, region, d['H'], null, "住宅", "百度")
|
||||
}
|
||||
|
||||
if (gaodeRest && gaodeRest.score >= 0.5) {
|
||||
d['P'] = gaodeRest.district
|
||||
d['Q'] = gaodeRest.name
|
||||
d['R'] = gaodeRest.address
|
||||
d['S'] = gaodeRest.lng
|
||||
d['T'] = gaodeRest.lat
|
||||
d['U'] = gaodeRest.type
|
||||
d['v'] = gaodeRest.source
|
||||
}
|
||||
}
|
||||
|
||||
if(!d['O']){
|
||||
def locationMap = getLocation(city, d['E'])
|
||||
if(locationMap){
|
||||
def mapRets = findNearestByMap(locationMap.lat, locationMap.lng)
|
||||
if(mapRets){
|
||||
def o = mapRets.get(0)
|
||||
d['W'] = o.cityname
|
||||
d['X'] = o.adname
|
||||
d['Y'] = o.name
|
||||
if(o.address && o.address instanceof String){
|
||||
d['Z'] = o.address
|
||||
}
|
||||
d['AA'] = o.type
|
||||
d['AB'] = o.distance
|
||||
d['AC'] = o.location
|
||||
}
|
||||
def dbRets = findNearestDistrict(city, locationMap.lat, locationMap.lng, 1000, 10)
|
||||
if(dbRets){
|
||||
def b = dbRets.get(0)
|
||||
d['AD'] = b.region
|
||||
d['AE'] = b.name
|
||||
d['AF'] = b.address
|
||||
d['AG'] = b.distance
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// }
|
||||
|
||||
},0);
|
||||
}
|
||||
|
||||
def distanceRun(def fileName){
|
||||
ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
def city = d['J']
|
||||
if(!d['O'] && d['E'] && d['E'].toString().length()>6){
|
||||
def locationMap = getLocation(city, d['E'])
|
||||
if(locationMap){
|
||||
def mapRets = findNearestByMap(locationMap.lat, locationMap.lng)
|
||||
if(mapRets){
|
||||
def o = mapRets.get(0)
|
||||
d['W'] = o.cityname
|
||||
d['X'] = o.adname
|
||||
d['Y'] = o.name
|
||||
if(o.address && o.address instanceof String){
|
||||
d['Z'] = o.address
|
||||
}
|
||||
d['AA'] = o.type
|
||||
d['AB'] = o.distance
|
||||
d['AC'] = o.location
|
||||
}
|
||||
def dbRets = findNearestDistrict(city, locationMap.lat, locationMap.lng, 1000, 10)
|
||||
if(dbRets){
|
||||
def b = dbRets.get(0)
|
||||
d['AD'] = b.region
|
||||
d['AE'] = b.name
|
||||
d['AF'] = b.address
|
||||
d['AG'] = b.distance
|
||||
}
|
||||
}
|
||||
}
|
||||
},0);
|
||||
}
|
||||
|
||||
def nearDistRun(def fileName){
|
||||
ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
def city = d['B']
|
||||
// if(!d['I'] && !d['J']){
|
||||
// city = d['B']
|
||||
// }
|
||||
if(d['O'] && d['O']=="周边"){
|
||||
def locationMap = null
|
||||
if(d['B'] && d['E']){
|
||||
locationMap = getLocation(city, d['E'])
|
||||
}
|
||||
// if(!locationMap && d['C']){
|
||||
// locationMap = getLocation(city, d['C'])
|
||||
// }
|
||||
|
||||
if(locationMap){
|
||||
def dbRets = findNearestDistrict(city, locationMap.lat, locationMap.lng, 1000, 10)
|
||||
if(dbRets){
|
||||
def b = dbRets.get(0)
|
||||
d['K'] = b.city
|
||||
d['L'] = b.region
|
||||
d['M'] = b.name
|
||||
d['N'] = b.address
|
||||
d['P'] = b.distance
|
||||
}
|
||||
}
|
||||
}
|
||||
},0);
|
||||
}
|
||||
|
||||
def run2V1(def fileName){
|
||||
ExcelUtils.multiThreadingModify(fileName,{ d->
|
||||
def city = d['J']
|
||||
if(d['O']){
|
||||
d['AH'] = d['L']
|
||||
d['AI'] = d['M']
|
||||
d['AJ'] = d['N']
|
||||
}
|
||||
if(!d['O']){
|
||||
if(d['R'] && d['R'].contains("号")){
|
||||
def addr = d['R'].toString().substring(0, d['R'].indexOf("号"))
|
||||
if((addr && d['E'] && d['E'].contains(addr))){
|
||||
def region = RegionData.formatRegion(city, d['P'])
|
||||
if(d['U'] && d['U'].toString().contains("住宅")){
|
||||
d['AH'] = region
|
||||
d['AI'] = d['Q']
|
||||
d['AJ'] = d['R']
|
||||
d['AK'] = d['U']
|
||||
d['AM'] = d['S']+","+d['T']
|
||||
d['O'] ="地图匹配"
|
||||
}else if(!d['U']){
|
||||
d['AH'] = region
|
||||
d['AI'] = d['Q']
|
||||
d['AJ'] = d['R']
|
||||
d['AK'] = d['U']
|
||||
d['AM'] = d['S']+","+d['T']
|
||||
d['O'] ="地图匹配"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(!d['O']){
|
||||
if(d['Y'] && d['AE']){
|
||||
def mapName = d['Y'].toString().replace(".", "").replace("(", "").replace(")", "").replace("。", "")
|
||||
.replace("·", "").replace("-", "").replace("`", "").replace("?", "").replace("‘", "")
|
||||
.replace("1期", "一期").replace("2期", "二期").replace("3期", "三期")
|
||||
.replace("4期", "四期").replace("5期", "五期").replace("12区", "十二区").replace("1区", "一区").replace("2区", "二区").replace("3区", "三区")
|
||||
.replace("4区", "四区").replace("5区", "五区").replace("a区", "A区")
|
||||
.replace("B区", "B区").replace("c区", "C区").replace("d区", "D区")
|
||||
if(mapName == d['AE']){
|
||||
def region = RegionData.formatRegion(city, d['AD'])
|
||||
d['AH'] = region
|
||||
d['AI'] = d['AE']
|
||||
d['AJ'] = d['AF']
|
||||
d['AK'] = d['DB']
|
||||
d['O'] ="名称相同"
|
||||
}else{
|
||||
if(d['AB'].toString().toDouble()>=d['AG'].toString().toDouble()){
|
||||
def region = RegionData.formatRegion(city, d['AD'])
|
||||
d['AH'] = region
|
||||
d['AI'] = d['AE']
|
||||
d['AJ'] = d['AF']
|
||||
d['AK'] = "DB库近"
|
||||
d['AL'] = d['AG']
|
||||
d['O'] ="距离比较"
|
||||
}else{
|
||||
def region = RegionData.formatRegion(city, d['X'])
|
||||
d['AH'] = region
|
||||
d['AI'] = d['Y']
|
||||
d['AJ'] = d['Z']
|
||||
d['AK'] = d['AA']
|
||||
d['AL'] = d['AB']
|
||||
d['AM'] = d['AC']
|
||||
d['O'] ="距离比较"
|
||||
}
|
||||
}
|
||||
}else if(d['Y']){
|
||||
def region = RegionData.formatRegion(city, d['X'])
|
||||
d['AH'] = region
|
||||
d['AI'] = d['Y']
|
||||
d['AJ'] = d['Z']
|
||||
d['AK'] = d['AA']
|
||||
d['AL'] = d['AB']
|
||||
d['AM'] = d['AC']
|
||||
d['O'] ="采用地图相近"
|
||||
}else if(d['AE']){
|
||||
def region = RegionData.formatRegion(city, d['AD'])
|
||||
d['AH'] = region
|
||||
d['AI'] = d['AE']
|
||||
d['AJ'] = d['AF']
|
||||
d['AK'] = "DB库近"
|
||||
d['AL'] = d['AG']
|
||||
d['O'] ="采用DB相近"
|
||||
}
|
||||
}
|
||||
},0);
|
||||
}
|
||||
|
||||
|
||||
def findNearestByMap(lng, lat){
|
||||
def rets = GaoDeMapApiData.doSearchByGaoDe("120000|120203|120300|120301|120302|120303|120304", lng, lat, 1000, 1, 10)
|
||||
return rets
|
||||
|
||||
}
|
||||
|
||||
public List<DBObject> findNearestDistrict(def city, double lat, double lng, int radiusMeters, int limit) {
|
||||
DBObject districtFields = new BasicDBObject("city", 1).append("region", 1).append("name", 1).append("address", 1)
|
||||
districtFields.put("estate_type2", 1)
|
||||
districtFields.put("lat", 1)
|
||||
districtFields.put("lng", 1)
|
||||
|
||||
|
||||
DBObject sdq = new BasicDBObject();
|
||||
sdq.put("sd.point", lat + "," + lng);
|
||||
sdq.put("sd.sfield", "lat_lng");
|
||||
sdq.put("sd.distance", String.valueOf((double) radiusMeters / 1000.0));
|
||||
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("cat", "district");
|
||||
q.put("city", city);
|
||||
q.put("visible", 0);
|
||||
q.put('$sd', sdq);
|
||||
|
||||
List<DBObject> list = new ArrayList<DBObject>()
|
||||
Iterator<DBObject> iter = SEAWEED.find(q, districtFields).limit(limit * 2).iterator()
|
||||
while(iter.hasNext()) {
|
||||
def o = iter.next()
|
||||
o.distance = GeoHelper.GetDistance(o.lat/10000000000.0, o.lng/10000000000.0, lat, lng)
|
||||
list.add(o)
|
||||
}
|
||||
|
||||
if (list.size() == 0) {
|
||||
return Collections.EMPTY_LIST;
|
||||
}
|
||||
list.sort{o1,o2->
|
||||
return o1.distance <=> o2.distance
|
||||
}
|
||||
if (list.size() > limit) {
|
||||
list = list.subList(0, limit);
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
def getLocation(city, addr){
|
||||
|
||||
if(addr){
|
||||
def rMap = [:]
|
||||
def reps = GaoDeApiUtils.geo(city, addr);
|
||||
if(reps && reps.count.toInteger()>=1){
|
||||
def map = reps.geocodes.get(0)
|
||||
def location = map.location
|
||||
def level = map.level
|
||||
if(location){
|
||||
def locMap = location.split(",")
|
||||
// def lat = dist.lat/10000000000.0;
|
||||
// def lng = dist.lng/10000000000.0;
|
||||
// def distance = GeoHelper.GetDistance(locMap[1].toDouble(), locMap[0].toDouble(), lat, lng)
|
||||
// rMap.distance = distance
|
||||
// rMap.level = level
|
||||
rMap.lat = locMap[1].toDouble()
|
||||
rMap.lng = locMap[0].toDouble()
|
||||
// rMap.keyword = map.formatted_address
|
||||
// rMap.location = location
|
||||
// rMap.region = map.district
|
||||
return rMap;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
|
||||
def run3(def fileName){
|
||||
ExcelUtils.multiThreadingModify(fileName,{ d->
|
||||
def city = d['I']
|
||||
def region = d['J']
|
||||
|
||||
def newName = null
|
||||
if(d['C']){
|
||||
if(d['C'].toString().length()<2){
|
||||
newName = null
|
||||
}else{
|
||||
newName = d['C'].toString().replace(".", "").replace("(", "").replace(")", "").replace("。", "")
|
||||
.replace("·", "").replace("-", "").replace("`", "").replace("?", "").replace("‘", "")
|
||||
.replace("1期", "一期").replace("2期", "二期").replace("3期", "三期")
|
||||
.replace("4期", "四期").replace("5期", "五期").replace("12区", "十二区").replace("a区", "A区")
|
||||
.replace("B区", "B区").replace("c区", "C区").replace("d区", "D区")
|
||||
}
|
||||
|
||||
}
|
||||
def address = null
|
||||
if(d['H']){
|
||||
address = d['H']
|
||||
}else{
|
||||
def arrs = d['B'].split("->")
|
||||
if(arrs && arrs.size()==4){
|
||||
address = arrs[3]
|
||||
}else if(arrs && arrs.size()==2){
|
||||
address = arrs[1]
|
||||
}
|
||||
}
|
||||
if(!region ||(region && d['L'] && region.contains(d['L']))) {
|
||||
if (d['B'] && !d['O'] && d['M']) {
|
||||
if (d['M'] == newName) {
|
||||
d['O'] = "相同"
|
||||
}
|
||||
if (!d['O']) {
|
||||
def dbo = CommonDatautils.findDistrictByAlias(d['K'], d['L'], newName, "district")
|
||||
if (dbo) {
|
||||
d['O'] = "别名匹配"
|
||||
}
|
||||
}
|
||||
|
||||
if (!d['O']) {
|
||||
def py1 = com.fangjia.core.util.DataUtil.hanyuQuanpin(newName, " ")
|
||||
def py2 = com.fangjia.core.util.DataUtil.hanyuQuanpin(d['M'], " ")
|
||||
if (py1 == py2) {
|
||||
d['O'] = "拼音相同"
|
||||
}
|
||||
}
|
||||
|
||||
if (!d['O'] && d['M'] && newName) {
|
||||
def word = null
|
||||
if (newName.toString().length() >= d['M'].toString().length()) {
|
||||
word = newName.toString().replace(d['M'], "")
|
||||
} else if (newName.toString().length() < d['M'].toString().length()) {
|
||||
word = d['M'].toString().replace(newName, "")
|
||||
}
|
||||
if (word) {
|
||||
def words = ["重庆", "小区", "(高层)", "花园", "恒都", "(洋房)", "社区"]
|
||||
for (def w : words) {
|
||||
if (word == w) {
|
||||
d['O'] = "去部分词后相同"
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if (!d['O'] && !word.endsWith("区") && !word.endsWith("期") && !word.endsWith("团") && !word.endsWith("东苑") && !word.endsWith("西苑") && !word.endsWith("南苑") && !word.endsWith("北苑")) {
|
||||
if (d['N'] && d['N'].toString().endsWith("号") && address.toString().contains(d['N'])) {
|
||||
|
||||
def score = CosineSimilarAlgorithm.getSimilarity(d['M'], newName);
|
||||
if (score >= 0.65) {
|
||||
println newName + " " + d['M'] + " " + score
|
||||
d['O'] = "地址相同"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!d['O'] && newName && !newName.contains("期") && !newName.endsWith("组团") && (!newName.endsWith("区") || newName.endsWith("小区"))) {
|
||||
|
||||
def gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, address, "住宅", "高德")
|
||||
if (!gaodeRest) {
|
||||
gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, address, "住宅", "百度")
|
||||
}
|
||||
if (gaodeRest && gaodeRest.name) {
|
||||
def mapName = gaodeRest.name.replace(".", "").replace("(", "").replace(")", "").replace("·", "").replace("`", "").replace("?", "")
|
||||
if (mapName == d['M'] && gaodeRest.score >= 0.7) {
|
||||
d['O'] = gaodeRest.source + "匹配"
|
||||
println newName + " " + d['M'] + " " + gaodeRest.score
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
if(newName && !d['O']){
|
||||
def gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, address, "住宅", "高德")
|
||||
def mapName = null
|
||||
if(gaodeRest && gaodeRest.name){
|
||||
mapName = gaodeRest.name.replace(".", "").replace("(", "").replace(")", "").replace("·", "").replace("`", "").replace("?", "")
|
||||
}
|
||||
if(!gaodeRest || (gaodeRest && mapName != newName)) {
|
||||
gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, d['E'], "住宅", "百度")
|
||||
if(gaodeRest && gaodeRest.name){
|
||||
mapName = gaodeRest.name.replace(".", "").replace("(", "").replace(")", "").replace("·", "").replace("`", "").replace("?", "")
|
||||
}
|
||||
}
|
||||
if(gaodeRest && mapName== newName){
|
||||
d['P'] = gaodeRest.district
|
||||
d['Q'] = mapName
|
||||
d['R'] = gaodeRest.address
|
||||
d['S'] = gaodeRest.lng
|
||||
d['T'] = gaodeRest.lat
|
||||
d['U'] = gaodeRest.type
|
||||
d['v'] = gaodeRest.source
|
||||
}
|
||||
}
|
||||
|
||||
},0);
|
||||
}
|
||||
|
||||
def mapRun(fileName){
|
||||
ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
def city = d['J']
|
||||
def region = d['I']
|
||||
if(city && region && d['Q'] && d['O']!="距离比较" && d['O']!="地址匹配"){
|
||||
def gaodeRest = FangjiaClientUtils.getMapData(city, region, d['Q'], null, "住宅", "高德")
|
||||
if(gaodeRest){
|
||||
d['V'] = gaodeRest.city
|
||||
d['W'] = gaodeRest.district
|
||||
d['X'] = gaodeRest.name
|
||||
d['Y'] = gaodeRest.address
|
||||
d['Z'] = gaodeRest.lng
|
||||
d['AA'] = gaodeRest.lat
|
||||
d['AB'] = gaodeRest.type
|
||||
}
|
||||
|
||||
d['AC'] = NameProcess.nameStandard(city,region, d['Q'])
|
||||
}
|
||||
},0)
|
||||
}
|
||||
}
|
||||
+89
@@ -0,0 +1,89 @@
|
||||
package com.fangjia.script.gj.chongqingblank
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.QueryOperators
|
||||
|
||||
class TestPrice {
|
||||
|
||||
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
static main(args) {
|
||||
def task = new TestPrice()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\估值1-V2.xlsx",{ d->
|
||||
if(d['K'] && d['M']){
|
||||
def dbo = findThirdPrice(d['K'], d['L'], d['M'], 202012, "lianjia_esf_price")
|
||||
if(dbo){
|
||||
d['AU'] = dbo.lianjia_esf_price
|
||||
}
|
||||
|
||||
def district = CommonDatautils.findDistrict(d['K'], d['L'], d['M'], "district")
|
||||
def startDate = 20201101
|
||||
def endDate = 20210601
|
||||
def trades = findTrades(district, startDate, endDate)
|
||||
if(trades){
|
||||
d['AV'] = (int)(trades.sum()/trades.size())
|
||||
d['AW'] = trades.size()
|
||||
}
|
||||
}
|
||||
},0);
|
||||
}
|
||||
|
||||
def findThirdPrice(def city, def region, def name, def s_date,def fieldName){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", name)
|
||||
q.put("s_date", new BasicDBObject(QueryOperators.GTE, s_date))
|
||||
q.put(fieldName, new BasicDBObject(QueryOperators.EXISTS, true))
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date",-1))
|
||||
def obj = null;
|
||||
for(def d : dbcursor){
|
||||
obj = d;
|
||||
break;
|
||||
}
|
||||
return obj;
|
||||
}
|
||||
|
||||
def findTrades(dist, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", dist.city)
|
||||
q.put("region", dist.region)
|
||||
// q.put("source", "链家在线")
|
||||
|
||||
def alias = []
|
||||
if(dist.alias){
|
||||
if(!dist.alias.contains(dist.name)){
|
||||
dist.alias << dist.name
|
||||
}
|
||||
alias = dist.alias
|
||||
}else{
|
||||
alias << dist.name
|
||||
}
|
||||
|
||||
if(alias){
|
||||
q.put("district_name", new BasicDBObject('$in', alias))
|
||||
}
|
||||
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
|
||||
def dbcorsor = DEAL_PRICE_COLL.find(q)
|
||||
def prices = []
|
||||
for(def d : dbcorsor){
|
||||
if(d.avg_price){
|
||||
prices << d.avg_price
|
||||
}
|
||||
}
|
||||
|
||||
return prices
|
||||
}
|
||||
|
||||
}
|
||||
+87
-20
@@ -1,39 +1,106 @@
|
||||
package com.fangjia.script.gj.data
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
class FieldStat {
|
||||
|
||||
final static DBCollection CITY_INFO_CONN = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", MongoUtils.MongoDB_Domain_mongo);
|
||||
|
||||
static main(args) {
|
||||
def task = new FieldStat();
|
||||
task.run()
|
||||
// task.run()
|
||||
task.run1()
|
||||
// task.run2()
|
||||
}
|
||||
|
||||
def run1(){
|
||||
def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
|
||||
def dist_total = 0
|
||||
|
||||
def addr_alias_total = 0
|
||||
def rets = []
|
||||
rets << ["城市","别名总量","有别名小区数量","小区总量","有别名小区占比"]
|
||||
for(def city : citys){
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias", "addr_alias"]);
|
||||
def alias_index = 0
|
||||
def alias_total = 0
|
||||
|
||||
for(def d : districts){
|
||||
def alias = d.alias
|
||||
if(alias){
|
||||
alias_index++
|
||||
alias_total = alias_total + alias.size()
|
||||
}
|
||||
}
|
||||
def precent = DataHandleUtils.getDoubleRound(2, alias_index*100.0/districts.size())
|
||||
rets << [city, alias_total, alias_index, districts.size(), precent+"%"]
|
||||
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\92个城市别名统计.xlsx", rets, "统计")
|
||||
// println dist_total
|
||||
// println alias_total
|
||||
// println addr_alias_total
|
||||
}
|
||||
|
||||
def run(){
|
||||
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\½¨ÐÐÊý¾ÝÈںϽá¹ûͳ¼Æ_0425.xlsx", { row ->
|
||||
def city = row['A']
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\城市统计.xlsx", { row ->
|
||||
def city = row['B']
|
||||
if(city){
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias"]);
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
|
||||
def total = districts.size()
|
||||
row['D'] = total
|
||||
|
||||
def aliasIndex = 0
|
||||
def aliasTotalCount = 0
|
||||
for(def d : districts){
|
||||
def alias = d.alias
|
||||
if(alias){
|
||||
aliasIndex++
|
||||
aliasTotalCount = aliasTotalCount + alias.size()
|
||||
}
|
||||
}
|
||||
// row['J'] = aliasTotalCount
|
||||
// row['K'] = aliasIndex
|
||||
def p = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, aliasIndex*100/total)
|
||||
row['B'] = p+"%"
|
||||
// row['M'] = total
|
||||
println city +" "+p
|
||||
// def aliasIndex = 0
|
||||
// def aliasTotalCount = 0
|
||||
// for(def d : districts){
|
||||
// def alias = d.alias
|
||||
// if(alias){
|
||||
// aliasIndex++
|
||||
// aliasTotalCount = aliasTotalCount + alias.size()
|
||||
// }
|
||||
// }
|
||||
//// row['J'] = aliasTotalCount
|
||||
//// row['K'] = aliasIndex
|
||||
// def p = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, aliasIndex*100/total)
|
||||
// row['B'] = p+"%"
|
||||
//// row['M'] = total
|
||||
// println city +" "+p
|
||||
}
|
||||
}, 1);
|
||||
}, 0);
|
||||
println "finish"
|
||||
}
|
||||
|
||||
def run2(){
|
||||
def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
|
||||
def rets = []
|
||||
rets << ["城市","建筑年代覆盖率","物业类型覆盖率","建筑类型覆盖率","区域最高小区均价覆盖率","区域最低小区均价覆盖率"]
|
||||
for(def c : citys){
|
||||
def obj = findCoverPrecent(c)
|
||||
if(obj){
|
||||
def datePrecent = DataHandleUtils.getDoubleRound(2, obj.district_detail.finishBuildingDate*100)
|
||||
def estateTypePrecent = DataHandleUtils.getDoubleRound(2, obj.district_detail.estateType2*100)
|
||||
def buildTypePrecent = DataHandleUtils.getDoubleRound(2, obj.district_detail.buildType*100)
|
||||
rets << [c, datePrecent+"%", estateTypePrecent+"%", buildTypePrecent+"%", "100%", "100%"]
|
||||
}
|
||||
}
|
||||
|
||||
ExcelUtils.write("d:\\数据\\92个城市覆盖率统计.xlsx", rets, "统计")
|
||||
|
||||
}
|
||||
|
||||
def findCoverPrecent(def city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
def dbcursor = CITY_INFO_CONN.find(q).sort(new BasicDBObject("weekend", -1)).limit(1)
|
||||
def obj = null
|
||||
for(def d : dbcursor){
|
||||
obj = d
|
||||
}
|
||||
return obj
|
||||
}
|
||||
}
|
||||
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
package com.fangjia.script.gj.data
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
class RegionExport {
|
||||
|
||||
static main(args) {
|
||||
def task = new RegionExport()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
ExcelUtils.multiThreadingModify("d:\\Êý¾Ý\\Ç×ÁÚ\\Ð¡Çø³ÇÊÐÁбí.xls",{d->
|
||||
def city = d['B']
|
||||
def newCity = RegionData.formatCity(city)
|
||||
if(newCity){
|
||||
def regions = CommonUtils.getAllRegions(newCity, "region")
|
||||
def regionStr = ""
|
||||
regions.each{k,v->
|
||||
regionStr = regionStr + k+"\n"
|
||||
}
|
||||
d['C'] = regionStr
|
||||
}
|
||||
},0)
|
||||
}
|
||||
}
|
||||
+3
-3
@@ -24,8 +24,8 @@ class ExportData {
|
||||
|
||||
static main(args) {
|
||||
def task = new ExportData();
|
||||
// task.exportThird()
|
||||
task.exportMap();
|
||||
task.exportThird()
|
||||
// task.exportMap();
|
||||
}
|
||||
|
||||
def exportMap(){
|
||||
@@ -71,7 +71,7 @@ class ExportData {
|
||||
|
||||
|
||||
def exportThird(){
|
||||
def citys = ["╩╢╟╡","ю╔и╫","а╛тф╦ш","до╬╘","дом╗","кчг╗","кужщ","л╘жщ","нчнЩ","пЛжщ","янЁг","яОжщ","уР╫╜"]
|
||||
def citys = ["ÖØÇì"]
|
||||
for(def city : citys){
|
||||
def rets = []
|
||||
rets << ["城市","区域","名称","别名"]
|
||||
|
||||
+36
-1
@@ -3,13 +3,18 @@ package com.fangjia.script.gj.district
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import net.sf.json.JSONArray
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
class FangjiaClientUtils {
|
||||
|
||||
public static def getMapData(def city, def region, def name, def address, def types, def mapCode){
|
||||
JSONObject jsonObj = null
|
||||
try {
|
||||
def url = "http://192.168.0.19:38080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&address="+address+"&types="+types+"&mapCode="+mapCode
|
||||
def url = "http://192.168.0.19:38080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&types="+types+"&mapCode="+mapCode
|
||||
//def url = "http://192.168.10.166:8080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&types="+types+"&mapCode="+mapCode
|
||||
if(address){
|
||||
url = url + "&address="+address
|
||||
}
|
||||
// def url = "http://localhost:8080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&address="+address+"&types="+types+"&mapCode="+mapCode
|
||||
def content = new HttpRequest().newGet(url, null, "utf-8")
|
||||
jsonObj = JSONObject.fromObject(content)
|
||||
@@ -44,4 +49,34 @@ class FangjiaClientUtils {
|
||||
|
||||
println content
|
||||
}
|
||||
|
||||
|
||||
public static def searchMapData(def city, def keyword, def types, def mapCode){
|
||||
def jsonObj = null
|
||||
try {
|
||||
def url = "http://192.168.0.19:38080/FangJiaDataService/search?city="+city+"&keyword="+keyword+"&types="+types+"&mapCode="+mapCode+"&limit=10"
|
||||
def content = new HttpRequest().newGet(url, null, "utf-8")
|
||||
// def ret = content.replace("[","").replace("]","")
|
||||
jsonObj = JSONArray.fromObject(content)
|
||||
} catch (Exception e) {
|
||||
return null;
|
||||
}
|
||||
return jsonObj;
|
||||
}
|
||||
|
||||
|
||||
public static Map<String, Object> JsonToMap(JSONObject json) {
|
||||
Map<String, Object> columnValMap = new HashMap<String, Object>();
|
||||
Set<Object> jsonKeys = json.keySet();
|
||||
for (Object key : jsonKeys) {
|
||||
Object JsonValObj = json.get(key);
|
||||
if (key instanceof JSONObject) {
|
||||
columnValMap.put((String) key, JsonToMap((JSONObject) JsonValObj));
|
||||
} else {
|
||||
columnValMap.put((String) key, JsonValObj);
|
||||
}
|
||||
}
|
||||
return columnValMap;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+236
@@ -0,0 +1,236 @@
|
||||
package com.fangjia.script.gj.es
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.apache.lucene.queryparser.classic.QueryParser
|
||||
|
||||
class EsMonthHouseStat {
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
static main(args) {
|
||||
def task = new EsMonthHouseStat()
|
||||
// task.run()
|
||||
task.tradeRun()
|
||||
// task.houseRun()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
|
||||
def rets = []
|
||||
def dates = ["2020-05","2020-06","2020-07","2020-08","2020-09","2020-10","2020-11","2020-12","2021-01","2021-02","2021-03","2021-04","2021-05"]
|
||||
def headers = ["城市"]
|
||||
headers.addAll(dates)
|
||||
rets << headers
|
||||
for(def city : citys){
|
||||
def datas = []
|
||||
datas << city
|
||||
for(def d : dates){
|
||||
def date = DateUtils.parseFromTightYYYYMMDD(d+"-01")
|
||||
def startDate = DateUtils.formatDateByPattern(date, "yyyyMMdd")
|
||||
def endDate = DateUtils.addMonth(date, 1).replace("-","")
|
||||
def count = findHouseCount(city, startDate.toInteger(), endDate.toInteger())
|
||||
datas << count
|
||||
}
|
||||
rets << datas
|
||||
println datas
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\建行92个城市房源统计.xlsx", rets, "统计")
|
||||
}
|
||||
|
||||
def tradeRun(){
|
||||
def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
|
||||
def rets = []
|
||||
def rets2 = []
|
||||
//,
|
||||
//"2021-01","2021-02","2021-03","2021-04","2021-05"
|
||||
//def dates = ["2020-01","2020-02","2020-03","2020-04","2020-05","2020-06","2020-07","2020-08","2020-09","2020-10","2020-11","2020-12"]
|
||||
def dates = ["2021-04"]
|
||||
def headers = ["城市"]
|
||||
headers.addAll(dates)
|
||||
rets << headers
|
||||
|
||||
|
||||
def headers2 = ["城市"]
|
||||
headers2.addAll(dates)
|
||||
rets2 << headers2
|
||||
for(def city : citys){
|
||||
def datas = []
|
||||
datas << city
|
||||
|
||||
def datas2 = []
|
||||
datas2 << city
|
||||
for(def d : dates){
|
||||
def startDate = DateUtils.parseFromTightYYYYMMDD(d+"-01")
|
||||
def endDateStr = DateUtils.addMonth(startDate, 1).replace("-","")
|
||||
def endDate = DateUtils.parseFromTightYYYYMMDD2(endDateStr)
|
||||
//def count = findTrades(city, startDate, endDate)
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias"]);
|
||||
def oldCount = 0
|
||||
def newCount = 0
|
||||
for(def dist : districts){
|
||||
def countMap = findTradesDetail(dist, startDate, endDate)
|
||||
oldCount = oldCount + countMap.total_old
|
||||
newCount = newCount + countMap.total_new
|
||||
println dist.city + " " + dist.name +" "+countMap.total_old+" "+countMap.total_new
|
||||
}
|
||||
|
||||
datas << oldCount
|
||||
datas2 << newCount
|
||||
println city +" "+ oldCount +" "+newCount
|
||||
}
|
||||
rets << datas
|
||||
|
||||
rets2 << datas2
|
||||
println datas
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\建行92个城市成交统计(去重前匹配).xlsx", rets, "统计")
|
||||
|
||||
ExcelUtils.write("d:\\数据\\建行92个城市成交统计(去重后匹配).xlsx", rets2, "统计")
|
||||
}
|
||||
|
||||
def houseRun(){
|
||||
ExcelUtils.multiThreadingModify("d:\\数据\\小区案例数据覆盖率.xlsx",{d->
|
||||
def city = d['B']
|
||||
if(city == "湖州"){
|
||||
def count = findHouseCount(city, 20210301, 20210601)
|
||||
d['E'] = count
|
||||
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias"]);
|
||||
def disthouseCount = 0
|
||||
for(def dist : districts){
|
||||
def alias = []
|
||||
if(dist.alias){
|
||||
if(!dist.alias.contains(dist.name)){
|
||||
dist.alias << dist.name
|
||||
}
|
||||
alias = dist.alias
|
||||
}else{
|
||||
alias << dist.name
|
||||
}
|
||||
|
||||
def distCount = findHouseCount(dist.city, dist.region, alias, 20210301, 20210601)
|
||||
if(distCount){
|
||||
disthouseCount++
|
||||
}
|
||||
println dist.city+" "+dist.region+" "+dist.name+" "+distCount
|
||||
}
|
||||
|
||||
println "disthouseCount:"+disthouseCount
|
||||
println districts.size()
|
||||
def precent = DataHandleUtils.getDoubleRound(2, disthouseCount*100.0/districts.size())
|
||||
d['D'] = precent+"%"
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
|
||||
def findHouseCount(def city, def region, def alias, def startDate, def endDate){
|
||||
def aliasStr = null;
|
||||
def total = 0
|
||||
try{
|
||||
aliasStr = escapeExprSpecialWord(alias.join("','"))
|
||||
String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}')"
|
||||
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
total = data.c
|
||||
})
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
|
||||
return total
|
||||
}
|
||||
|
||||
/**
|
||||
* 转义字符串中的特殊字符
|
||||
* 仅过滤Lucene regexps, 其他的勿用
|
||||
* @param keyword
|
||||
* @return
|
||||
*/
|
||||
public static String escapeExprSpecialWord(String keyword) {
|
||||
return QueryParser.escape(keyword);
|
||||
}
|
||||
|
||||
def findHouseCount(def city, def startDate, def endDate){
|
||||
|
||||
String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} "
|
||||
def total = 0
|
||||
ElasticUtils.executeSqlQuery(sql,{ data->
|
||||
total = data.c
|
||||
})
|
||||
return total
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def findTrades(city, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
// q.put("region", dist.region)
|
||||
// q.put("source", "链家在线")
|
||||
|
||||
// def alias = []
|
||||
// if(dist.alias){
|
||||
// if(!dist.alias.contains(dist.name)){
|
||||
// dist.alias << dist.name
|
||||
// }
|
||||
// alias = dist.alias
|
||||
// }else{
|
||||
// alias << dist.name
|
||||
// }
|
||||
|
||||
// if(alias){
|
||||
// q.put("district_name", new BasicDBObject('$in', alias))
|
||||
// }
|
||||
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
|
||||
def count = DEAL_PRICE_COLL.count(q)
|
||||
return count
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findTradesDetail(dist, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", dist.city)
|
||||
q.put("region", dist.region)
|
||||
|
||||
def alias = []
|
||||
if(dist.alias){
|
||||
if(!dist.alias.contains(dist.name)){
|
||||
dist.alias << dist.name
|
||||
}
|
||||
alias = dist.alias
|
||||
}else{
|
||||
alias << dist.name
|
||||
}
|
||||
|
||||
if(alias){
|
||||
q.put("district_name", new BasicDBObject('$in', alias))
|
||||
}
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
def total = dbcursor.count()
|
||||
def rets = new HashSet()
|
||||
for(def d : dbcursor){
|
||||
def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
|
||||
if(!rets.contains(key)){
|
||||
rets << key
|
||||
}
|
||||
}
|
||||
def rMap = [:]
|
||||
rMap.total_old = total
|
||||
rMap.total_new = rets.size()
|
||||
return rMap
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
package com.fangjia.script.gj.es
|
||||
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
|
||||
class EsSelectTest {
|
||||
EsDataSource dataSource = new EsDataSource();
|
||||
String address = "jdbc:es://http://192.168.0.86:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s";
|
||||
|
||||
static main(args) {
|
||||
def task = new EsSelectTest()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("request", "/ws/czbank"))
|
||||
//.must(QueryBuilders.termQuery("request_body", "findSample"))
|
||||
|
||||
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.86", "fangjia-log*", queryBuilder);
|
||||
if(houses) {
|
||||
for (int k = 0; k < houses.size(); k++) {
|
||||
println k
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def endDate = new Date()
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
dataSource.setProperties(connectionProperties)
|
||||
StringBuffer sqlBuffer = new StringBuffer();
|
||||
String d = DateUtil.formatByPattern(new Date(), "yyyy.MM.dd")
|
||||
long startD = DateUtil.getBeforeMinute(new Date(), -7).getTime();
|
||||
long endD = DateUtil.getBeforeMinute(endDate, 1).getTime();
|
||||
println(DateUtil.formatByPattern(new Date(startD), "yyyy-MM-dd HH:mm:ss")+"===="+DateUtil.formatByPattern(new Date(endD), "yyyy-MM-dd HH:mm:ss"));
|
||||
|
||||
|
||||
sqlBuffer.append("select request_time FROM \"fangjia-log*\" WHERE 1=1");
|
||||
sqlBuffer.append(" and request='/ws/czbank' and request_body='findSample'")
|
||||
|
||||
ResultSet rs = null
|
||||
try {
|
||||
Connection connection = dataSource.getConnection()
|
||||
Statement statement = connection.createStatement()
|
||||
rs = statement.executeQuery(sqlBuffer.toString())
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
StringBuffer msgBuffer = new StringBuffer();
|
||||
while (rs.next()) {
|
||||
|
||||
def responsetime = rs.getLong("responsetime");
|
||||
def timestamp = DateUtil.formatByPattern(new Date(rs.getLong("time")), "yyyy-MM-dd HH:mm")
|
||||
println responsetime
|
||||
}
|
||||
}
|
||||
}
|
||||
+171
@@ -0,0 +1,171 @@
|
||||
package com.fangjia.script.gj.location
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.looplineinfo.Point
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import java.text.SimpleDateFormat
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
|
||||
class LocationTest {
|
||||
|
||||
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
|
||||
|
||||
static main(args) {
|
||||
def task = new LocationTest()
|
||||
// task.run()
|
||||
task.run2()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def regionPointsMap = findRegionRange()
|
||||
def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region", "name", "address", "lat", "lng"]);
|
||||
def index = 0
|
||||
def rets = []
|
||||
for(def dist : districts){
|
||||
def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, regionPointsMap.get(dist.region))
|
||||
if(!flag){
|
||||
index++
|
||||
|
||||
def ret = []
|
||||
ret << dist.city
|
||||
ret << dist.region
|
||||
ret << dist.name
|
||||
|
||||
def loactionMap = matchMap(dist)
|
||||
if(loactionMap){
|
||||
ret << "地图"
|
||||
ret << loactionMap.region
|
||||
ret << loactionMap.mapName
|
||||
def distance = GeoHelper.GetDistance(dist.lat/10000000000.0, dist.lng/10000000000.0, loactionMap.lat, loactionMap.lng)
|
||||
ret << distance
|
||||
if(dist.region==loactionMap.region){
|
||||
println index+" "+dist.region +" "+dist.name +" 不在区域范围内,与地图相同"
|
||||
}else{
|
||||
println index+" "+dist.region +" "+dist.name +" 不在区域范围内 地图:"+ loactionMap.region+" "+distance
|
||||
}
|
||||
}else{
|
||||
ret << ""
|
||||
println index+" "+dist.region +" "+dist.name +" 不在区域范围内,地图没找到"
|
||||
}
|
||||
rets << ret
|
||||
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\福州\\区域经纬度错误3.xlsx", rets, "区域不在行政范围内")
|
||||
}
|
||||
|
||||
def run2(){
|
||||
ExcelUtils.multiThreadingModify("d:\\数据\\福州\\福州小区.xlsx")
|
||||
}
|
||||
|
||||
def matchMap(def dist){
|
||||
def gaodeRest = FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "高德")
|
||||
def locationMap = null
|
||||
if(gaodeRest){
|
||||
locationMap = getMapLocation(dist, gaodeRest);
|
||||
if(locationMap){
|
||||
return locationMap
|
||||
}
|
||||
}
|
||||
|
||||
def baiduRest = null
|
||||
if (!locationMap) {
|
||||
baiduRest = FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "百度")
|
||||
}
|
||||
if(baiduRest){
|
||||
locationMap = getMapLocation(dist, baiduRest);
|
||||
}
|
||||
return locationMap
|
||||
}
|
||||
|
||||
def getMapLocation(def dist, def rets){
|
||||
def lat = null
|
||||
def lng = null
|
||||
def region = null
|
||||
def mapName = null
|
||||
for(def r : rets){
|
||||
if(!r.city){
|
||||
continue
|
||||
}
|
||||
if(!r.city.contains(dist.city)){
|
||||
continue
|
||||
}
|
||||
def name = r.name.replace("?","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("【","").replace("】","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期").replace(" ","").trim()
|
||||
if(dist.name == name){
|
||||
lat = r.lat
|
||||
lng = r.lng
|
||||
mapName = r.name
|
||||
region = r.district
|
||||
}else{
|
||||
def word = null
|
||||
if (name.length() >= dist.name.length()) {
|
||||
word = name.replace(dist.name, "")
|
||||
} else if (name.length() < dist.name.length()) {
|
||||
word = dist.name.replace(name, "")
|
||||
}
|
||||
if (word) {
|
||||
def words = ["商住楼", "小区","公寓","别墅"]
|
||||
for (def w : words) {
|
||||
if (word == w) {
|
||||
lat = r.lat
|
||||
lng = r.lng
|
||||
region = r.district
|
||||
mapName = r.name
|
||||
break
|
||||
}
|
||||
}
|
||||
if(!lng){
|
||||
Pattern pattern = Pattern.compile("\\d+号楼");
|
||||
Matcher matcher = pattern.matcher(word);
|
||||
if(matcher.find()){
|
||||
lat = r.lat
|
||||
lng = r.lng
|
||||
region = r.district
|
||||
mapName = r.name
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if(lng){
|
||||
break
|
||||
}
|
||||
}
|
||||
if(lng){
|
||||
return ["region":region, "lng":lng, "lat":lat, "mapName":mapName]
|
||||
}
|
||||
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
def isInArea(lng, lat, points){
|
||||
List<Point> pointList = InsidePolygon.createPoint2(points);
|
||||
//Point point = new Point(lng, lat);
|
||||
//def flag = InsidePolygon.aboutZone(point, pointList)
|
||||
def flag = InsidePolygon.isPtInPoly(lng, lat, pointList)
|
||||
return flag;
|
||||
}
|
||||
|
||||
def findRegionRange(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city","福州市")
|
||||
def dbcursor = POLYLINECOLL.find(q)
|
||||
def regionMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def region = RegionData.formatRegion("福州",d.region)
|
||||
regionMap.put(region, d.polyline)
|
||||
}
|
||||
return regionMap
|
||||
}
|
||||
}
|
||||
+15
-13
@@ -4,6 +4,7 @@ import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
class CrawlDistrictIdSelect {
|
||||
@@ -14,8 +15,8 @@ class CrawlDistrictIdSelect {
|
||||
|
||||
static main(args) {
|
||||
def task = new CrawlDistrictIdSelect()
|
||||
task.run()
|
||||
// task.synch()
|
||||
// task.run()
|
||||
task.synch()
|
||||
}
|
||||
|
||||
def synch(){
|
||||
@@ -50,13 +51,13 @@ class CrawlDistrictIdSelect {
|
||||
|
||||
def findCrawlData(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city","¸£ÖÝ")
|
||||
q.put("source", "°²¾Ó¿Í")
|
||||
q.put("s_date",new BasicDBObject('$gte', 202001))
|
||||
def dbcursor = thirdPartyColl.find(q)
|
||||
// q.put("city","¸£ÖÝ")
|
||||
q.put("source", "Öî¸ðÕÒ·¿")
|
||||
q.put("s_date",202010)
|
||||
def dbcursor = thirdPartyColl.find(q).batchSize(10000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def retMap = [:]
|
||||
def prefix = "/community/view/"
|
||||
def keyList =[]
|
||||
def prefix = "xiaoqu.zhuge.com/"
|
||||
def keyList =new HashSet()
|
||||
for(def db : dbcursor){
|
||||
def url = null
|
||||
if(!db.url){
|
||||
@@ -67,12 +68,11 @@ class CrawlDistrictIdSelect {
|
||||
url = db.url
|
||||
}
|
||||
if(url && url.contains(prefix) ){
|
||||
def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.size())
|
||||
def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.size()-1)
|
||||
if(district_id=="0"){
|
||||
continue
|
||||
}
|
||||
def shortCity = url.substring(url.indexOf("https://")+8, url.indexOf(".anjuke.com"))
|
||||
def region = RegionData.formatRegion(db.city, db.region)
|
||||
def shortCity = url.substring(url.indexOf("//")+2, url.indexOf(".xiaoqu.zhuge.com"))
|
||||
def key = shortCity +"#"+district_id
|
||||
|
||||
def ids = retMap.get(key)
|
||||
@@ -83,15 +83,17 @@ class CrawlDistrictIdSelect {
|
||||
|
||||
if(!keyList.contains(key)){
|
||||
def map = new BasicDBObject()
|
||||
map.root = "anjuke.com"
|
||||
map.root = "zhuge.com"
|
||||
map.city_short= shortCity
|
||||
map.city = db.city
|
||||
map.region = region
|
||||
map.region = db.region
|
||||
map.district_id = district_id
|
||||
map.source_url = url
|
||||
ids << map
|
||||
|
||||
keyList << key
|
||||
}else{
|
||||
println "ÖØ¸´£º"+key
|
||||
}
|
||||
}else{
|
||||
println url +" Á´½Ó²»ºÏÒªÇó"
|
||||
|
||||
+56
@@ -32,6 +32,22 @@ class InsidePolygon {
|
||||
|
||||
return list;
|
||||
}
|
||||
|
||||
|
||||
public static List<Point> createPoint2(String points){
|
||||
String[] array = points.split(";");
|
||||
List<Point> list = new ArrayList<Point>();
|
||||
for(int i =0 ;i<array.length-1;i++){
|
||||
def pointArrs = array[i].split("[,|\\|]")
|
||||
Point point = new Point();
|
||||
double x = Double.parseDouble(pointArrs[0]);
|
||||
double y = Double.parseDouble(pointArrs[1]);
|
||||
point.setLon(x);
|
||||
point.setLat(y);
|
||||
list.add(point);
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
public static List<Map<String, Object>> getPoints(String points){
|
||||
String[] array = points.split("-");
|
||||
@@ -167,4 +183,44 @@ class InsidePolygon {
|
||||
return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
|
||||
public static boolean isPtInPoly (double ALon , double ALat , List<Point> ps) {
|
||||
int iSum, iCount, iIndex;
|
||||
double dLon1 = 0, dLon2 = 0, dLat1 = 0, dLat2 = 0, dLon;
|
||||
if (ps.size() < 3) {
|
||||
return false;
|
||||
}
|
||||
iSum = 0;
|
||||
iCount = ps.size();
|
||||
for (iIndex = 0; iIndex<iCount;iIndex++) {
|
||||
if (iIndex == iCount - 1) {
|
||||
dLon1 = ps[iIndex].getLon()
|
||||
dLat1 = ps[iIndex].getLat()
|
||||
dLon2 = ps[0].getLon()
|
||||
dLat2 = ps[0].getLat()
|
||||
} else {
|
||||
dLon1 = ps[iIndex].getLon()
|
||||
dLat1 = ps[iIndex].getLat()
|
||||
dLon2 = ps[iIndex + 1].getLon()
|
||||
dLat2 = ps[iIndex + 1].getLat()
|
||||
}
|
||||
// 以下语句判断A点是否在边的两端点的水平平行线之间,在则可能有交点,开始判断交点是否在左射线上
|
||||
if (((ALat >= dLat1) && (ALat < dLat2)) || ((ALat >= dLat2) && (ALat < dLat1))) {
|
||||
if (Math.abs(dLat1 - dLat2) > 0) {
|
||||
//得到 A点向左射线与边的交点的x坐标:
|
||||
dLon = dLon1 - ((dLon1 - dLon2) * (dLat1 - ALat) ) / (dLat1 - dLat2);
|
||||
// 如果交点在A点左侧(说明是做射线与 边的交点),则射线与边的全部交点数加一:
|
||||
if (dLon < ALon) {
|
||||
iSum++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if ((iSum % 2) != 0) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
+13
-1
@@ -1,6 +1,9 @@
|
||||
package com.fangjia.script.temp.shenjian.poi
|
||||
|
||||
import com.alibaba.fastjson.JSONObject
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.statistics.poi.AmapPoi
|
||||
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
|
||||
import com.fangjia.script.temp.shenjian.util.HttpUtils
|
||||
@@ -21,7 +24,8 @@ import java.util.concurrent.Executors
|
||||
* @author shenjian
|
||||
* @since 2021/5/24
|
||||
*/
|
||||
class SeaweedAroundPoi {
|
||||
@BatchOwner(Watcher.ShenJian)
|
||||
class SeaweedAroundPoi extends BaseBatch {
|
||||
|
||||
static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017)
|
||||
static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi"
|
||||
@@ -170,4 +174,12 @@ class SeaweedAroundPoi {
|
||||
put("ÂÃÓξ°µã","110000");
|
||||
put("ͼÊé¹Ý","140500");
|
||||
}};
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
def types = listAllType()
|
||||
//dealSpecialSeaweedAround(types)
|
||||
dealPoi(new BasicDBObject(), types)
|
||||
executorService.shutdown()
|
||||
}
|
||||
}
|
||||
|
||||
+10
-5
@@ -139,12 +139,17 @@ class GaoDeUtils {
|
||||
ex.printStackTrace()
|
||||
}
|
||||
} else if (json && json.infocode && json.infocode == "10044") {
|
||||
//throw new RuntimeException("开发者的日访问量超限,被系统自动封停,第二天0:00会自动解封")
|
||||
// 多线程情况下还是exit有效果,抛异常不会停止程序
|
||||
if (keyUsed.intValue() >= GAO_DE_AK.size()) {
|
||||
System.exit(1)
|
||||
synchronized (this) {
|
||||
for (int i = 0; i < GAO_DE_AK.size(); i++) {
|
||||
if (GAO_DE_AK[i] == params.key) {
|
||||
if (i + 1 > GAO_DE_AK.size()) {
|
||||
System.exit(1)
|
||||
}
|
||||
keyUsed = new AtomicInteger(i+1)
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
keyUsed.incrementAndGet()
|
||||
} else if (json && json.infocode && json.infocode == "10004") {
|
||||
// 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。
|
||||
Thread.sleep(60000)
|
||||
|
||||
+5
@@ -25,6 +25,7 @@ class CrawalDBUtil {
|
||||
static DBCollection data_process_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "data_process")
|
||||
static DBCollection average_statistics_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "average_statistics")
|
||||
static DBCollection map_district_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "map_district_process")
|
||||
static DBCollection districtDetailColl = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_detail")
|
||||
|
||||
public static getSeedsCol() {
|
||||
return seedsColl;
|
||||
@@ -67,6 +68,10 @@ class CrawalDBUtil {
|
||||
return map_district_col;
|
||||
}
|
||||
|
||||
public static getDistrictDetailColl(){
|
||||
return districtDetailColl;
|
||||
}
|
||||
|
||||
static def updateSeedsCount(def id, def count) {
|
||||
|
||||
DBObject uquery = new BasicDBObject()
|
||||
|
||||
+1
-1
@@ -61,7 +61,7 @@ class DataStandardBatch extends BaseBatch{
|
||||
def static final SHORT_DATE = Utils.getLastMonth()
|
||||
def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6]
|
||||
|
||||
def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房"]
|
||||
def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
static {
|
||||
thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
|
||||
+106
@@ -0,0 +1,106 @@
|
||||
package com.fangjia.script.thirdPriceCrawl
|
||||
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.apache.solr.common.util.Hash
|
||||
|
||||
class SynchDistrictPrice {
|
||||
|
||||
def static final SHORT_DATE = Utils.getLastMonth()
|
||||
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
|
||||
def sourceMap = ["jiwu.com":"吉屋"]
|
||||
//"58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","jiwu.com":"吉屋"
|
||||
|
||||
static main(args) {
|
||||
def task = new SynchDistrictPrice()
|
||||
task.run()
|
||||
}
|
||||
|
||||
|
||||
def run(){
|
||||
def citys = ["福州"]//CommonUtils.findAllCity()
|
||||
|
||||
sourceMap.each { k, v ->
|
||||
for(def city : citys){
|
||||
def rets = findCrawlDistrictDetails(city, k, v)
|
||||
synchData(city, rets, v)
|
||||
println city +" "+rets.size()
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def findCrawlDistrictDetails(city, root, source){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("root", root)
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("city", 1)
|
||||
keys.put("region", 1)
|
||||
keys.put("block", 1)
|
||||
keys.put("name", 1)
|
||||
keys.put("address", 1)
|
||||
keys.put("avg_price", 1)
|
||||
keys.put("root", 1)
|
||||
keys.put("url", 1)
|
||||
def rets = []
|
||||
def dbcursor = districtDetailColl.find(q, keys)
|
||||
def names = new HashSet()
|
||||
for(def d : dbcursor){
|
||||
def s = d.city +"#"+d.region+"#"+d.name
|
||||
if(!names.contains(s)){
|
||||
names << s
|
||||
d.put("district_id", d.url)
|
||||
d.put("source", source)
|
||||
d.put("s_date", SHORT_DATE)
|
||||
d.removeField("root")
|
||||
d.removeField("url")
|
||||
d.removeField("_id")
|
||||
rets << d
|
||||
}
|
||||
}
|
||||
return rets
|
||||
}
|
||||
|
||||
def synchData(def city, def rets, def source){
|
||||
def q = new BasicDBObject()
|
||||
q.put("source", source)
|
||||
q.put("s_date", SHORT_DATE)
|
||||
q.put("city", city)
|
||||
def count = thirdPartyColl.count(q)
|
||||
def addIndex = 0
|
||||
if(count==0){
|
||||
try{
|
||||
thirdPartyColl.insert(rets)
|
||||
}catch(Exception e){
|
||||
println e.getMessage()
|
||||
}
|
||||
|
||||
}else{
|
||||
for(def r : rets){
|
||||
def flag = findIsExists(r.city, r.region, r.name, SHORT_DATE, source)
|
||||
if(!flag){
|
||||
addIndex++
|
||||
thirdPartyColl.save(r)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
println "["+source +"-"+city + "] add:"+ addIndex+" 原抓取:"+count
|
||||
}
|
||||
|
||||
def findIsExists(def city, def region, def name, def sDate, def source){
|
||||
def q = new BasicDBObject()
|
||||
q.put("source", source)
|
||||
q.put("s_date", sDate)
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", name)
|
||||
def count = thirdPartyColl.count(q)
|
||||
if(count){
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -30,7 +30,7 @@ class ThirdDataToGatherBatch extends BaseBatch{
|
||||
"房估估":"fanggugu_esf","诸葛找房":"zhugezhaofang_esf","安居客":"anjuke_esf","搜房":"soufang_esf","中原地产":"zhongyuan_esf","赶集":"ganji_esf",
|
||||
"链家":"lianjia_esf","兔博士":"tuboshi_esf","城市房产":"cityhouse_esf","贝壳找房":"beike_esf","Q房":"qfang_esf","Q房(新)":"qfang_new","中原地产(新)":"zhongyuan_new",
|
||||
"乐有家":"leyoujia_esf","乐有家(新)":"leyoujia_new","房探":"fangtan_esf","房多多":"fangdoudou_esf","小资家":"xiaozijia_esf","我爱我家":"5i5j_esf","麦田":"maitian_esf",
|
||||
"房途":"fangtoo_esf","C21":"c21_esf","乐居":"leju_esf","房产超市":"fccs_esf","58同城":"58tc_esf","365淘房":"365house_esf"
|
||||
"房途":"fangtoo_esf","C21":"c21_esf","乐居":"leju_esf","房产超市":"fccs_esf","58同城":"58tc_esf","365淘房":"365house_esf","吉屋":"jiwu_esf"
|
||||
]
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(2)
|
||||
|
||||
|
||||
@@ -309,6 +309,31 @@ public class OpenApiClient {
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
public static JSONObject addressStandard(String city, String address) {
|
||||
if (StringUtils.isBlank(city) || StringUtils.isBlank(address)) {
|
||||
throw new IllegalArgumentException();
|
||||
}
|
||||
|
||||
try {
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("city", city);
|
||||
params.put("address", address);
|
||||
params.put("token", TOKEN);
|
||||
String content = new HttpRequest().get("http://open.fangjia.com/address/standard", params, "utf-8")
|
||||
JSONObject result = JSONObject.fromObject(content)
|
||||
if (result.getInt("code") == 200) {
|
||||
return result.getJSONObject("result");
|
||||
} else if (result.getInt("code") == 500) {
|
||||
println("${city} ${address} ${result}")
|
||||
}
|
||||
} catch (Exception e) {
|
||||
println("${city} ${address} ${e.getMessage()}")
|
||||
e.printStackTrace();
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
public static JSONObject evaluateByGeo(String city, def lat, def lng, String buildDate, String estateType, String geoType) {
|
||||
try {
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
@@ -1214,4 +1239,6 @@ public class OpenApiClient {
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import com.fangjia.core.ProvinceData
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DataUtil
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
@@ -11,45 +12,71 @@ import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
DBCollection SEAWEED_TOWNSHIP = MongoUtils.getCollectionByName("fangjia_base", "seaweed_township", "192.168.0.235", 27018)
|
||||
def count = 0
|
||||
List<List<String>> data = Collections.synchronizedList([])
|
||||
data << ["ID","城市","区域","小区名","经度","纬度","街道"]
|
||||
|
||||
CommonUtils.findAllCity().each {def city->
|
||||
DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject item->
|
||||
if(StringUtils.isBlank(item.township) && item.lng){
|
||||
count++
|
||||
def lng = item.lng/10000000000d
|
||||
def lat = item.lat/10000000000d
|
||||
def location = "${lng},${lat}"
|
||||
JSONObject resp = GaoDeApiUtils.geocode(location)
|
||||
def row = []
|
||||
row << item._id.toString()
|
||||
row << item.city
|
||||
row << item.region
|
||||
row << item.name
|
||||
row << lng
|
||||
row << lat
|
||||
BasicDBObject dbo = new BasicDBObject()
|
||||
dbo.id = item._id.toString()
|
||||
dbo.city = item.city
|
||||
dbo.region = item.region
|
||||
dbo.name = item.name
|
||||
dbo.lng = lng
|
||||
dbo.lat = lat
|
||||
if(resp.regeocode && resp.regeocode.addressComponent && resp.regeocode.addressComponent.township){
|
||||
row << resp.regeocode.addressComponent.township
|
||||
dbo.township = resp.regeocode.addressComponent.township
|
||||
}
|
||||
println(row)
|
||||
data << row
|
||||
SEAWEED_TOWNSHIP.insert(dbo)
|
||||
}
|
||||
})
|
||||
}
|
||||
ExcelUtils.write("D:\\街道补充数据.xlsx", data, "补充数据")
|
||||
//DBCollection SEAWEED_TOWNSHIP = MongoUtils.getCollectionByName("fangjia_base", "seaweed_township", "192.168.0.235", 27018)
|
||||
//def count = 0
|
||||
//List<List<String>> data = Collections.synchronizedList([])
|
||||
//data << ["ID","城市","区域","小区名","经度","纬度","街道"]
|
||||
//
|
||||
//CommonUtils.findAllCity().each {def city->
|
||||
// DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject item->
|
||||
// if(StringUtils.isBlank(item.township) && item.lng){
|
||||
// count++
|
||||
// def lng = item.lng/10000000000d
|
||||
// def lat = item.lat/10000000000d
|
||||
// def location = "${lng},${lat}"
|
||||
// JSONObject resp = GaoDeApiUtils.geocode(location)
|
||||
// def row = []
|
||||
// row << item._id.toString()
|
||||
// row << item.city
|
||||
// row << item.region
|
||||
// row << item.name
|
||||
// row << lng
|
||||
// row << lat
|
||||
// BasicDBObject dbo = new BasicDBObject()
|
||||
// dbo.id = item._id.toString()
|
||||
// dbo.city = item.city
|
||||
// dbo.region = item.region
|
||||
// dbo.name = item.name
|
||||
// dbo.lng = lng
|
||||
// dbo.lat = lat
|
||||
// if(resp.regeocode && resp.regeocode.addressComponent && resp.regeocode.addressComponent.township){
|
||||
// row << resp.regeocode.addressComponent.township
|
||||
// dbo.township = resp.regeocode.addressComponent.township
|
||||
// }
|
||||
// println(row)
|
||||
// data << row
|
||||
// SEAWEED_TOWNSHIP.insert(dbo)
|
||||
// }
|
||||
// })
|
||||
//}
|
||||
//ExcelUtils.write("D:\\街道补充数据.xlsx", data, "补充数据")
|
||||
|
||||
//ExcelUtils.modify("D:\\街道补充数据.xlsx",{List item ->
|
||||
// if(item["G"] && item["H"] == null){
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id", item["A"]), new BasicDBObject('$set', new BasicDBObject('township', item["G"])))
|
||||
// }
|
||||
// println(item)
|
||||
//})
|
||||
|
||||
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Downloads\\错误经纬度.xlsx",{List item->
|
||||
// def location = "${item["A"]},${item["B"]}"
|
||||
// JSONObject resp = GaoDeApiUtils.geocode(location)
|
||||
// if(resp.regeocode && resp.regeocode.formatted_address){
|
||||
// item['G'] = resp.regeocode.formatted_address
|
||||
// println(item)
|
||||
// }
|
||||
//})
|
||||
|
||||
//def count = 0
|
||||
//CommonUtils.findAllCity().each {def city->
|
||||
// DbUtils.executeQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject item->
|
||||
// if(item.township){
|
||||
// count++
|
||||
// }
|
||||
// }, new BasicDBObject("township", 1))
|
||||
//}
|
||||
//println(count)
|
||||
|
||||
ElasticUtils.executeSqlQuery("select count(*), p_date from seawater where p_date > 20210501 group by p_date",{
|
||||
println(it)
|
||||
})
|
||||
@@ -9,6 +9,11 @@ import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import net.sf.json.JSONObject
|
||||
|
||||
def strt(String addr){
|
||||
|
||||
}
|
||||
|
||||
|
||||
ExcelUtils.modify("D:\\nextcloud\\数据\\地址类服务测试\\建行地址相似度评级测试.xlsx",{List item->
|
||||
def city = item["A"]
|
||||
def address1 = item["B"]
|
||||
|
||||
+26
@@ -223,6 +223,32 @@ class GaoDeMapApiData {
|
||||
doSearchByGaoDe(type, lng, lat, radius, page)
|
||||
}
|
||||
}
|
||||
|
||||
def static doSearchByGaoDe(String type, Double lng, Double lat, int radius, int page, int pageSize) {
|
||||
Map<String, String> params = new HashMap<String, String>()
|
||||
params.put("key", GAO_DE_AK)
|
||||
params.put("types", type)
|
||||
params.put("location", Double.toString(lng)+ ","+ Double.toString(lat))
|
||||
params.put("radius",radius)
|
||||
params.put("offset",pageSize)
|
||||
params.put("page",page)
|
||||
String content
|
||||
try {
|
||||
content = new HttpRequest().newGet("http://restapi.amap.com/v3/place/around", params, "utf-8")
|
||||
} catch (Exception e) {
|
||||
content = new HttpRequest().newGet("http://restapi.amap.com/v3/place/around", params, "utf-8")
|
||||
}
|
||||
JSONObject jsonObj = JSONObject.fromObject(content)
|
||||
|
||||
if(jsonObj.get("infocode").equals("10000")){
|
||||
JSONArray pois = jsonObj.get("pois")
|
||||
return pois
|
||||
} else {
|
||||
println jsonObj.get("infocode")+"exit........"
|
||||
doSearchByGaoDe(type, lng, lat, radius, page, pageSize)
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
private synchronized static fzData(JSONArray pois) {
|
||||
pois.each { poi ->
|
||||
|
||||
Reference in New Issue
Block a user