diff --git a/deliverables/k8s/fangjia-bkoff.yaml b/deliverables/k8s/fangjia-bkoff.yaml index a6e01a7ab..768fd80cd 100644 --- a/deliverables/k8s/fangjia-bkoff.yaml +++ b/deliverables/k8s/fangjia-bkoff.yaml @@ -39,13 +39,15 @@ spec: name: web protocol: TCP readinessProbe: - tcpSocket: - port: 8080 + httpGet: + path: /k8s/readiness + port: web initialDelaySeconds: 5 periodSeconds: 10 livenessProbe: - tcpSocket: - port: 8080 + httpGet: + path: /k8s/liveness + port: web initialDelaySeconds: 10 periodSeconds: 5 volumeMounts: diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/K8sController.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/K8sController.java new file mode 100644 index 000000000..11a80b8a2 --- /dev/null +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/K8sController.java @@ -0,0 +1,28 @@ +package com.fangjia.bkoff.controllers; + +/** + * 房价网后台存活监控 + * + * @author shenjian + * @since 2021/6/7 + */ +public class K8sController { + + /** + * 存活检查 + * + * @return + */ + public Object livenessAction() { + return "liveness"; + } + + /** + * 就绪检查 + * + * @return + */ + public Object readinessAction() { + return "readiness"; + } +} diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/interceptor/AuthInterceptor.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/interceptor/AuthInterceptor.java index 192767d85..111b3c515 100644 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/interceptor/AuthInterceptor.java +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/interceptor/AuthInterceptor.java @@ -40,7 +40,8 @@ public class AuthInterceptor implements Interceptor { request.setAttribute("__resources", request.getSession().getAttribute(userId + "_resources")); if (invocation.getAction().getController().getName().equals("login") - || invocation.getAction().getController().getName().equals("")) { + || invocation.getAction().getController().getName().equals("") + || invocation.getAction().getController().getName().equals("k8s")) { return invocation.invoke(); } diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java index 3a6029b19..e71226a4a 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java @@ -237,7 +237,8 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl i } private JSONObject query(String id,String category, String city, String region, String name, List alias, - List regionAlias, Integer startDate, Integer endDate, Double startArea, Double endArea, Integer startPrice, Integer endPrice, Integer room, String root,String sort, boolean order, int start, int limit,String table){ + List regionAlias, Integer startDate, Integer endDate, Double startArea, Double endArea, Integer startPrice, Integer endPrice, + Integer room, String root,String sort, boolean order, int start, int limit,String table){ long time1 = System.currentTimeMillis(); JSONObject reqBody = new JSONObject(); JSONObject q = new JSONObject(); @@ -351,7 +352,7 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl i if(endPrice != null){ temp2.put("lte", endPrice); } - param.put("total_price", temp2); + param.put("avg_price", temp2); temp.put("range", param); jsonList.add(temp); } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GuzhiController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GuzhiController.java index 2f3621338..870dda5e3 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GuzhiController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GuzhiController.java @@ -156,12 +156,12 @@ public class GuzhiController { return mv; } - + //福州农商行人工录入 public Object checkAction(@Valid GuzhiCheckModel model){ RestfulModelAndView mv = new RestfulModelAndView(); mv.setModel(BasicModel.buildOKModel()); - String msg = "【房价网-人工审查地址】接收到新的人工审查地址信息,请求请及时处理!城市:" + model.getCity() + String msg = "【房价网-人工审查地址】接收到新的人工审查地址信息,请求请及时处理!城市:" + model.getCity()+" 区域:"+model.getDistrict() +" 小区名称:"+(model.getName()!=null?model.getName():"") + " 产证地址:" + model.getAddress(); DingDingMessageUtil.sendTextMessage("065f4a05d5f87eb39a9f9e048eeefb80c1f7560446f8926441de80453c160c60", msg); return mv; diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java index 93172390a..37b785212 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java @@ -671,7 +671,7 @@ public class PropertyController { && !"1007486".equals(userId) && !"1007533".equals(userId) && !"1007630".equals(userId) && !"1007705".equals(userId) && !"1007739".equals(userId) && !"1007736".equals(userId) && !"1007741".equals(userId) && !"1007742".equals(userId) && !"1007743".equals(userId) - && !"1007747".equals(userId) + && !"1007747".equals(userId) && !"1007757".equals(userId) && model.getDetailLevel() != null && model.getDetailLevel() != 0) {// 临时权限控制 mav.setModel(BasicModel.buildUnAuthorizedModel()); return mav; diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/model/PropertyEvaluationModel.java b/projects/fangjia-open/src/main/java/com/fangjia/open/model/PropertyEvaluationModel.java index 61f0618f7..5953fc3a3 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/model/PropertyEvaluationModel.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/model/PropertyEvaluationModel.java @@ -284,6 +284,8 @@ public class PropertyEvaluationModel extends FangjiaApiModel { private String user; + private Integer avgPrice; + public void setDecorationAge(int decorationAge) { this.decorationAge = decorationAge; @@ -895,4 +897,12 @@ public class PropertyEvaluationModel extends FangjiaApiModel { public void setAtticSize(Double atticSize) { this.atticSize = atticSize; } + + public Integer getAvgPrice() { + return avgPrice; + } + + public void setAvgPrice(Integer avgPrice) { + this.avgPrice = avgPrice; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/avgprice/MonthlyDistrictStatCalcBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/avgprice/MonthlyDistrictStatCalcBatch.groovy index e6daa5e94..1d1396141 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/avgprice/MonthlyDistrictStatCalcBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/avgprice/MonthlyDistrictStatCalcBatch.groovy @@ -30,12 +30,12 @@ public class MonthlyDistrictStatCalcBatch extends BaseBatch { void process(Object args) { List cities = []; def allCity = CommonUtils.findAllCity() - allCity = ["苏州"] + def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1) cities.addAll(allCity) def districts = [] cities.each {def city-> - districts.addAll DbUtils.seaweed.find(new BasicDBObject("city", city).append("cat", "district").append("visible", 0).append("name", "吾悦广场"), keys).batchSize(2000).toList() + districts.addAll DbUtils.seaweed.find(new BasicDBObject("city", city).append("cat", "district").append("visible", 0), keys).batchSize(2000).toList() println(city) } ExecutorService executorService = Executors.newFixedThreadPool(20) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/dealPrice/DealPriceMathBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/dealPrice/DealPriceMathBatch.groovy index d9e3c9364..5c5aadd9e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/dealPrice/DealPriceMathBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/dealPrice/DealPriceMathBatch.groovy @@ -54,6 +54,10 @@ class DealPriceMathBatch extends BaseBatch{ def flag = true; + + def startDate = DateUtil.getFirstDayOfMonth(new Date(),-1) + def endDate = DateUtil.getFirstDayOfMonth(new Date(), 1) + cityMap.each{city,alias-> println "开始加载【${city}】小区" if(city != "昆玉"){ @@ -61,10 +65,8 @@ class DealPriceMathBatch extends BaseBatch{ districtMap.putAll(detailMap) } } - def startDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2020-01-01","yyyy-MM-dd")) -// def startDate = DateUtil.getFirstDayOfMonth(new Date()) -// def endDate = DateUtil.getFirstDayOfMonth(new Date(), 1) - def endDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2021-04-30","yyyy-MM-dd")) +// def startDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2020-01-01","yyyy-MM-dd")) +// def endDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2021-04-30","yyyy-MM-dd")) while(flag){ println "开始加载解析数据......" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangjiatong/AccessLogBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangjiatong/AccessLogBatch.groovy index 0b4d0724b..06e84af84 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangjiatong/AccessLogBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangjiatong/AccessLogBatch.groovy @@ -211,23 +211,35 @@ class AccessLogBatch extends BaseBatch{ } def read(){ - String fileName = "C:\\Users\\gaojuan\\Desktop\\localhost_access_log.2019-06-01.txt"; + String fileName = "D:\\数据\\logs\\performance.open.log"; File file = new File(fileName); def rets = [] - file.eachLine{ + def index = 0 + def list =[] + def slowCount = 0 + file.eachLine{it-> - if(it.contains("/fangjiatong2/")){ - try{ - it = it.replace(it, URLDecoder.decode(URLDecoder.decode(it, "utf-8"), "UTF-8")) - }catch(IllegalArgumentException e){} + + if(it.contains("ws/czbank") && it.contains("entity:House")){ +// println it + index++ +// try{ + def s1 = it.substring(it.indexOf("time:")+5, it.length()) + s1 = s1.substring(0, s1.indexOf("\t")) + list << s1.toInteger() + if(s1.isNumber() && s1.toInteger()>=4000){ + println it + slowCount++ + } +// }catch(IllegalArgumentException e){} } - if(!it.contains("漕溪四村")){ - rets << it - } + } - - writeTxt(rets, "C:\\Users\\gaojuan\\Desktop\\fjt-2020-6-1.txt"); + println list.sum()/list.size() + println index + println "slowCount:"+slowCount +// writeTxt(rets, "C:\\Users\\gaojuan\\Desktop\\fjt-2020-6-1.txt"); } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy index 95e40fb9b..9934cb1c7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy @@ -102,7 +102,7 @@ class DistrictInfoExportBatch extends BaseBatch{ row << rMap.estate_company row << getBuildStructure(rMap.build_structure)//为空的,默认钢混 row << rMap.loudong_count - row << processAlias(rMap.alias) + row << processAlias(rMap.alias, rMap.addr_alias) def addr = fzAddrMap.get(rMap.get("_id").toString()); if(addr){ @@ -110,20 +110,32 @@ class DistrictInfoExportBatch extends BaseBatch{ }else{ row << rMap.address } + //row << processAlias(rMap.addr_alias) result << row } } return result } - def processAlias(alias){ + def processAlias(alias, def addr_alias){ + def newAlias = [] if(alias){ - def newAlias = [] for(def name : alias){ - if(name && !name.contains("(") && !name.contains("、") && !name.contains("?") && !name.contains("(")){ + if(name && !name.contains("(") && !name.contains("、") && !name.contains("?") && !name.contains("(") && !newAlias.contains(name)){ newAlias << name } } + } + + if(addr_alias){ + for(def name : addr_alias){ + if(name && !name.contains("(") && !name.contains("、") && !name.contains("?") && !name.contains("(") && !newAlias.contains(name)){ + newAlias << name + } + } + } + + if(newAlias){ return newAlias.join(","); } return null; @@ -165,6 +177,9 @@ class DistrictInfoExportBatch extends BaseBatch{ rets << "非钢混" } } + if(rets.size()==0){ + rets << "钢混" + } return rets.join(",") } return "钢混"; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy index c2ecb8014..87a2d9962 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy @@ -76,8 +76,8 @@ class DistrictPriceExportBatch extends BaseBatch{ def createDatas(def city){ def districts = CommonDatautils.findAllDistricts(city, "district", null); - def lastDate = DateUtils.getLastDayOfMonth(new Date(), -1) - def startDate = DateUtils.getFirstDayOfMonth(lastDate); + def lastDate = new Date() + def startDate = DateUtils.getFirstDayOfMonth(lastDate, -1) def result = [] for(def rMap : districts){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy new file mode 100644 index 000000000..cf1e4950f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -0,0 +1,133 @@ +package com.fangjia.script.fuzhou + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) +def count0 = 0 +def count1 = 0 +def count2 = 0 +def count3 = 0 +def count4 = 0 +def count5 = 0 +def count6 = 0 +def count7 = 0 +def count8 = 0 +def total=4943 +for(def d : datas){ + def list = [] + if(d['G']){ + list << d['G'] + } + if(d['H']){ + list << d['H'] + } + if(d['I']){ + list << d['I'] + } + if(d['J']){ + list << d['J'] + } + if(d['K']){ + list << d['K'] + } + if(d['L']){ + list << d['L'] + } + if(d['M']){ + list << d['M'] + } + if(d['N']){ + list << d['N'] + } + if(d['O']){ + list << d['O'] + } + if(list.size()==0){ + count0++ + } + if(list.size()==1){ + count1++ + } + if(list.size()==2){ + count2++ + } + if(list.size()==3){ + count3++ + } + if(list.size()==4){ + count4++ + } + if(list.size()==5){ + count5++ + } + + if(list.size()==6){ + count6++ + } + if(list.size()==7){ + count7++ + } + if(list.size()==8){ + count8++ + } +} + +println("count0:"+ (DataHandleUtils.getDoubleRound(2, count0*100.0/total))) +println("count1:"+ (DataHandleUtils.getDoubleRound(2, count1*100.0/total))) +println("count2:"+ (DataHandleUtils.getDoubleRound(2, count2*100.0/total))) +println("count3:"+ (DataHandleUtils.getDoubleRound(2, count3*100.0/total))) +println("count4:"+ (DataHandleUtils.getDoubleRound(2, count4*100.0/total))) +println("count5:"+ (DataHandleUtils.getDoubleRound(2, count5*100.0/total))) +println("count6:"+ (DataHandleUtils.getDoubleRound(2, count6*100.0/total))) +println("count7:"+ (DataHandleUtils.getDoubleRound(2, count7*100.0/total))) +println("count8:"+ (DataHandleUtils.getDoubleRound(2, count8*100.0/total))) + + + +////def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region", "name"]); +//def total = 8018 +//def names = ["诸葛找房":"zhugezhaofang_esf_price","安居客":"anjuke_esf_price","搜房":"soufang_esf_price","链家":"lianjia_esf_price","城市房产":"cityhouse_esf_price","房探":"fangtan_esf_price","房多多":"fangdoudou_esf_price","麦田":"maitian_esf_price", "58同城":"58tc_esf_price"] +// +//def rets = [] +//def sDate = 202011 +//rets << ["时间","诸葛找房","安居客","搜房","链家","城市房产","房探","房多多","麦田", "58同城"] +//def ret = [] +//ret << sDate +//names.each{k,v-> +// def crawlCount = crawlCount(k, sDate) +// def count = count(v, sDate) +// println k +" "+DataHandleUtils.getDoubleRound(2, count*100.0/total)+"("+crawlCount+")" +// ret << DataHandleUtils.getDoubleRound(2,count*100.0/total)+"("+crawlCount+")" +//} +//rets.add(ret) +//ExcelUtils.write("d:\\数据\\第三方均价覆盖率1.xlsx",rets,"数据") +// +//def crawlCount(k, sDate){ +// def thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); +// def q = new BasicDBObject() +// q.put("city","福州") +// q.put("s_date", sDate) +// q.put("source",k) +// return thirdPartyColl.count(q) +//} +// +// +//def count(name, sDate){ +// DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") +// def q = new BasicDBObject() +// q.put("city","福州") +// q.put("s_date", sDate) +// q.put(name, new BasicDBObject('$gt', 0)); +// return AVERAGE_STATISTICS.count(q) +//} + + + + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy index c08426b9c..6a36f086d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy @@ -18,7 +18,7 @@ import org.apache.commons.codec.digest.DigestUtils */ class ThirdPriceExportBatch extends BaseBatch{ - def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")); + def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") @@ -156,12 +156,12 @@ class ThirdPriceExportBatch extends BaseBatch{ if(!retMap.cityhouse_esf_price && d.cityhouse_esf_price){ retMap.cityhouse_esf_price = d.cityhouse_esf_price } - if(!retMap.fangtan_esf_price && d.fangtan_esf_price){ - retMap.fangtan_esf_price = d.fangtan_esf_price + if(!retMap.fangtan_esf_price && d.jiwu_esf_price){//使用吉屋替换房探 待抓取 + retMap.fangtan_esf_price = d.jiwu_esf_price } - if(!retMap.fangdoudou_esf_price && d.fangdoudou_esf_price){ - retMap.fangdoudou_esf_price = d.fangdoudou_esf_price + if(!retMap.fangdoudou_esf_price && d.get("365house_esf_price")){//使用淘房365替换房多多 + retMap.fangdoudou_esf_price = d.get("365house_esf_price") } if(!retMap.maitian_esf_price && d.maitian_esf_price){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy index 2ba2cdf53..e07d6eff0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/GaoDeMapCrawl.groovy @@ -3,6 +3,9 @@ package com.fangjia.script.gj import com.fangjia.core.ProvinceData import com.fangjia.core.RegionData import com.fangjia.httpclient.UserAgents +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher import com.fangjia.script.gj.paimai.GaoDeUtils import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.JacksonUtil @@ -30,39 +33,40 @@ import java.util.regex.Matcher import java.util.regex.Pattern import java.util.zip.GZIPInputStream -class GaoDeMapCrawl { +@BatchOwner(Watcher.GaoJuan) +class GaoDeMapCrawl extends BaseBatch{ DBCollection BOUNDS_COLL = MongoUtils.getCollectionByName("fangjia_base", "city_bounds_box", "192.168.0.235", 27018) - static def targetUrl = "http://192.168.0.206:9200/amap_poi/_doc" + def targetUrl = "http://192.168.0.206:9200/amap_poi/_doc" - public static final def dbUrl = "mongodb://192.168.0.235:27018"; + final def dbUrl = "mongodb://192.168.0.235:27018"; - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1) + ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1) final AtomicInteger count = new AtomicInteger() def updateLists = [] def updateErrorLists = [] def flagLists = [] def index = 0 - def prefix = "district_" + // def categorys = ["141202", "141203"]//中学,小学 - def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//住宅相关 + +// def prefix = "district_" +// def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//住宅相关 + // def categorys = ["190301"]//道路相关 + + def prefix = "village_" + def categorys = ["190108","190109"]//村庄 def totalCount = new AtomicLong() static main(args) { - def task = new GaoDeMapCrawl(); - -// def citys = CommonUtils.findAllCity() -// def i =1 -// for(def city : citys){ -// i++ -// println i+" "+ city -// } - - task.run() -// task.allRun(); + runBatch(new GaoDeMapCrawl(), args) } + @Override + void process(Object args) { + run() + } def allRun(){ def cityLations = null; def stop = false; @@ -110,14 +114,20 @@ class GaoDeMapCrawl { def run(){//,"西安","重庆" -// def citys = [ "临沂","泰安","菏泽","东营","德州","济宁","日照","聊城","滨州","枣庄","莱芜","济南","青岛","威海","烟台","潍坊","淄博"] def citys = ["重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] // def citys = CommonUtils.findAllCity() def totalCount = 0; for(def city : citys){ - def cityLations = findAllLation(city); - println "开始处理["+city +"]:"+cityLations.size() - gaoDeCitySearch(cityLations); + def stop = false + while(stop == false){ + def cityLations = findAllLation(city); + if(!cityLations){ + stop = true; + }else{ + println "开始处理["+city +"]:"+cityLations.size() + gaoDeCitySearch(cityLations); + } + } } } @@ -129,7 +139,7 @@ class GaoDeMapCrawl { q.put(prefix+"status", new BasicDBObject(QueryOperators.EXISTS, false)) // q.put("status", "失败") // q.put("traffic_status", new BasicDBObject(QueryOperators.EXISTS, false)) - def results = BOUNDS_COLL.find(q).batchSize(2000).addOption(Bytes.QUERYOPTION_NOTIMEOUT) + def results = BOUNDS_COLL.find(q).limit(2000).addOption(Bytes.QUERYOPTION_NOTIMEOUT) return results; } @@ -142,7 +152,7 @@ class GaoDeMapCrawl { // void run() { // try{ // totalCount.addAndGet(1) - searching(location) + searching(i, location) // }catch(Exception e){ // e.printStackTrace() // } finally { @@ -169,7 +179,7 @@ class GaoDeMapCrawl { } } - def searching(location){ + def searching(i, location){ // def bound_gd = location.bound_gd // def lats = [bound_gd[1], bound_gd[3]] // def lngs = [bound_gd[0], bound_gd[2]] @@ -177,7 +187,7 @@ class GaoDeMapCrawl { // lngs.sort() // println index // if(lngs[0]<=121.418153 && lngs[1]>=121.418153 && lats[0]<=31.218824 && lats[1]>=31.218824){ - search(location) + search(i, location) // } if(updateLists.size() > 0 && updateLists.size()>1000){ @@ -196,7 +206,7 @@ class GaoDeMapCrawl { } - def search(location){ + def search(i, location){ def bound_gd = location.bound_gd def lats = [bound_gd[1], bound_gd[3]] def lngs = [bound_gd[0], bound_gd[2]] @@ -230,7 +240,7 @@ class GaoDeMapCrawl { updateErrorLists.addAll(location) } // update(location); - println "[ "+totalCount.get()+" ]"+location._id+" "+location.city+" "+location.bound_gd+" flag: "+m.status + println "[ "+i+" ]"+location._id+" "+location.city+" "+location.bound_gd+" flag: "+m.status } def processData(rets){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy index 76ef3729b..4ec26d7db 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy @@ -43,6 +43,24 @@ class OpenApiUtils { } + static def matchDetailDistrict(def city, def address, def suggestDistrict, def preciseLevelflag){ + address = address.replace("(","(").replace(")",")").replace(" ","") + def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&suggestDistrict="+suggestDistrict+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1" + if(preciseLevelflag){ + url = url +"&preciseLevel=strict" + } + String content = new HttpRequest().get(url, "utf-8"); + if(content){ + def doc = new JsonSlurper().parseText(content) + if(doc && doc.code ==200 && doc.result){ + def dto = doc.result + return dto + } + } + return null; + } + + static def matchDistrict2(def city, def address, def preciseLevelflag){ address = address.replace("(","(").replace(")",")").replace(" ","").replace("#","") def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index 32b251e77..25c631832 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -47,7 +47,7 @@ class YouDaDataParser { def final AtomicInteger count_ = new AtomicInteger(0) - static def statDate = "2021-03-01" + static def statDate = "2021-04-01" static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo @@ -285,7 +285,7 @@ class YouDaDataParser { def esfParser(){ - def files = ["D:\\数据\\友达\\2021年4月\\二手房\\查询1.DBF"] + def files = ["D:\\数据\\友达\\2021年5月\\二手房\\查询1.DBF"] def datas = [] for(def fileName : files){ def rows = readDBF(fileName); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCityPriceStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCityPriceStat.groovy new file mode 100644 index 000000000..c60a811b5 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCityPriceStat.groovy @@ -0,0 +1,77 @@ +package com.fangjia.script.gj.ccb + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +class CcbCityPriceStat { + + public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; + public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) + + static main(args) { + def task = new CcbCityPriceStat() + task.run() + } + def run(){ + def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] + def rets = [] + for(def city : citys){ + def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "alias"]) + def priceMap = CommonDatautils.queryDistrictPrice(city, DateUtils.parseFromTightYYYYMMDD("2021-05-30")) + def cityRets = [] + for(def dist : districts){ + def tradeAvgPrice = findTrades(dist, "2021-02-01", "2021-06-01") + def weekendPrice = priceMap.get(city +"##"+dist.region+"##"+dist.name) + if(tradeAvgPrice && weekendPrice){ + def row = [city, dist.region, dist.name, weekendPrice, tradeAvgPrice.toInteger()] + cityRets << row + println row.toString() + } + } + print city +" "+ cityRets.size() + rets.addAll(cityRets) + } + + println "总量:"+rets.size() + ExcelUtils.write("d:\\数据\\92个城市价格数据.xlsx", rets, "所有") + } + + + def findTrades(dist, startDate, endDate){ + def q = new BasicDBObject() + q.put("city", dist.city) + q.put("region", dist.region) + + if(dist.alias){ + if(!dist.alias.contains(dist.name)){ + dist.alias << dist.name + } + q.put("district_name", new BasicDBObject('$in', dist.alias)) + }else{ + q.put("district_name", dist.name) + } + + q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate))) + def dbcorsor = DEAL_PRICE_COLL.find(q) + def avg_price = 0 + def total_price = 0 + def count = 0 + + for(def d : dbcorsor){ + if(d.avg_price){ + total_price = total_price + d.avg_price + count++ + } + } + if(count > 3){ + avg_price = total_price/count + } + return avg_price + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DataMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DataMatch.groovy new file mode 100644 index 000000000..9f84f6479 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DataMatch.groovy @@ -0,0 +1,322 @@ +package com.fangjia.script.gj.chongqingblank + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.util.OpenApiClient +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import net.sf.json.JSONObject +import com.fangjia.script.gj.* + +import java.util.regex.Matcher +import java.util.regex.Pattern + +class DataMatch { + + static final DBCollection FANGJIA_OFFICIAL_AREA_MAP = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "mongo.db.fangjia.com") + + + + static main(args) { + def task = new DataMatch() +// task.run1() + task.run2() +// task.run3() +// println task.findAddress("昌州街道棠香路") + +// task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx") + } + + def run1(){ + def fileName = "D:\\数据\\重庆农商行\\估值2-V5.xlsx" + //matchRegionValue(fileName); + matchDistrict(fileName) +// ExcelUtils.multiThreadingModify(fileName,{d-> +// if(d['O']=="新增"){ +// +// d['L'] = d['P'] +// d['M'] = d['Q'] +// d['N'] = d['R'] +// } +// },0) + + } + + def parseAddress(def fileName){ + ExcelUtils.multiThreadingModify(fileName,{d-> + + },0) + } + + def matchDistrict(fileName){ + ExcelUtils.multiThreadingModify(fileName,{d-> + def region = d['Q'] + def city = d['P'] +// if(!d['I'] && !d['J']){ +// city = d['B'] +// region = d['C'] +// } + if(d['R'] && !d['G']){ + def results = null +// if(d['E'] && isStreet(d['E'])){//加上效果并没有想象中的好 +// results = OpenApiClient.addressMatch(city, d['E'], "property", false, region, d['D']) +// } + if((results && results.precise<0) || !results){ + results = OpenApiClient.addressMatch(city, d['R'], "property", false, region, null) + } +// if((d['E'] && results && results.precise<0) || !results){ +// results = OpenApiClient.addressMatch(city, d['E'], "property", false, region, d['D']) +// } + if(results && results.precise>=0){ + d['K'] = results.city + d['L'] = results.district + d['M'] = results.name + d['N'] = results.address + } + } + },0); + } + + def run2(){ + def fileName = "D:\\数据\\重庆农商行\\估值2-V2.xlsx" +// matchRegionValue(fileName); + matchDistrict2(fileName) + } + + + def matchDistrict2(fileName){ + ExcelUtils.multiThreadingModify(fileName,{d-> + def region = d['I'] + def city = d['J'] + if(!d['I']){ + city = d['J'] + } + if(city && d['E'] && !d['O'] ){ + def addressResult = OpenApiClient.addressStandard(city, d['E']) + if(addressResult){ + if(addressResult.street && addressResult.streetNumber){ + d['H'] = addressResult.street+addressResult.streetNumber + } + } + def results = null + if(d['H']){ + results = OpenApiClient.addressMatch(city, d['H'], "property", false, region, null) + } + + if((d['E'] && results && results.precise<0) || !results){ + results = OpenApiClient.addressMatch(city, d['E'], "property", false, region, null) + } + + if(results && results.precise>=0){ + d['K'] = results.city + d['L'] = results.district + d['M'] = results.name + d['N'] = results.address + } + } + },0); + } + + def run3(){ + def fileName = "D:\\数据\\重庆农商行\\估值3.xlsx" +// matchDistrict3(fileName) + ExcelUtils.multiThreadingModify(fileName,{d-> + if(d['I']){ + d['I'] = d['I'].toString().replace("省","").replace("市","") + } + if(d['J'] && d['I']){ + d['J'] = RegionData.formatRegion(d['I'], d['J']) + } + },0) + } + + + def matchDistrict3(fileName){ + ExcelUtils.multiThreadingModify(fileName,{d-> + def address = d['B'] + if(address) { + def arrs = address.split("->") + def city = null + def region = null + def addr = null + if (arrs && arrs.size() == 4) { + city = arrs[0] + region = arrs[2] + addr = arrs[3] + d['I'] = city + d['J'] = region + } else if (arrs && arrs.size() == 2) { + def parseCity = CommonUtils.parseCityInAddress(arrs[1]) + city = parseCity + d['I'] = parseCity + addr = arrs[1] + } + if (addr && !d['O']) { + def addressResult = OpenApiClient.addressStandard(city, addr) + if (addressResult) { + if (!region) { + region = addressResult.district + d['J'] = region + } + if (addressResult.street && addressResult.streetNumber) { + d['H'] = addressResult.street + addressResult.streetNumber + } + } + def results = null + def name = null + if (d['C']) { + name = d['C'].toString().replace(".", "").replace("(", "").replace(")", "").replace("。", "") + .replace("·", "").replace("-", "").replace("`", "").replace("?", "").replace("‘", "") + .replace("1期", "一期").replace("2期", "二期").replace("3期", "三期") + .replace("4期", "四期").replace("5期", "五期").replace("12区", "十二区").replace("1区", "一区").replace("2区", "二区").replace("3区", "三区") + .replace("4区", "四区").replace("5区", "五区").replace("a区", "A区") + .replace("B区", "B区").replace("c区", "C区").replace("d区", "D区") + } + if (name) { + results = OpenApiClient.addressMatch(city, name, "property", false, region, null) + } + + if ((addr && results && results.precise < 0) || !results) { + results = OpenApiClient.addressMatch(city, addr, "property", false, region, name) + } + if (results && results.precise >= 0) { + d['K'] = results.city + d['L'] = results.district + d['M'] = results.name + d['N'] = results.address + } + } + } + + },0); + } + + + + def matchRegionValue(fileName){ + ExcelUtils.multiThreadingModify(fileName,{d-> + def code = d['C'] + if(code && code.toString().isNumber()){ + if(code.toInteger().toString()=="500226"){ + d['I'] = "荣昌" + d['J'] = "重庆" + }else if(code.toInteger().toString()=="500223"){ + d['I'] = "潼南" + d['J'] = "重庆" + }else{ + def data = findRegionCodeValue(code.toInteger().toString()); + if(data){ + d['I'] = data.fangjia_district + d['J'] = data.fangjia_city + }else{ + data = findCityCodeValue(code.toInteger().toString()) + if(data){ + d['J'] = data.fangjia_city + } + } + } + + } + + },0) + } + + def findCityCodeValue(value){ + def q = new BasicDBObject() + q.put("city_code", value) + def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q) + def count = dbcursor.count() + if(count >1){ + def data = null + for(def db : dbcursor){ + data = db + break + } + return data; + } + } + + def findRegionCodeValue(value){ + def q = new BasicDBObject() + q.put("district_code", value) + def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q) + def count = dbcursor.count() + if(count == 1){ + def data = null + for(def db : dbcursor){ + data = db + break + } + return data; + } + } + + def findAddress(String address) { + //Pattern ROAD_PATTERN = Pattern.compile('.*[路|大道|巷|胡同|街]\\d+号'); + Pattern ROAD_PATTERN = Pattern.compile('.[里,路,道,街,巷,路段,胡同].?\\d+.?\\d+[号,弄]|.[里,路,道,街,巷,路段,胡同].?\\d+.?[号,弄]') + Matcher m = ROAD_PATTERN.matcher(address) + String newAddress + if (m.find()) { + newAddress = m.group() + } + return newAddress + } + + def run2MatchV2(fileName){ + ExcelUtils.multiThreadingModify(fileName, { d -> + def region = d['I'] + def city = d['J'] + if(!d['I'] && !d['J']){ + city = d['B'] + region = d['C'] + } + if(d['D']) { + def name = d['D'].toString().replace(".", "").replace("(", "").replace(")", "").replace("。", "") + .replace("·", "").replace("-", "").replace("`", "").replace("?", "").replace("‘", "") + .replace("1期", "一期").replace("2期", "二期").replace("3期", "三期") + .replace("4期", "四期").replace("5期", "五期").replace("12区", "十二区").replace("1区", "一区").replace("2区", "二区").replace("3区", "三区") + .replace("4区", "四区").replace("5区", "五区").replace("a区", "A区") + .replace("B区", "B区").replace("c区", "C区").replace("d区", "D区") + + def result = OpenApiUtils.matchDetailDistrict(city, name, region, true) + if (!result && d['E']) { + if (findAddress(d['E'])) { + result = OpenApiUtils.matchDetailDistrict(city, d['E'], region, true) + } + } + if(result){ + def matchResult = result.matchResult + if (result.credit > 0.5 && result.matchResult && !result.matchResult.match) { + matchResult = amendMatch(result, d['E']) + } + if (matchResult.match) { + d['AQ'] = result.city + d['AR'] = result.district + d['AS'] = result.name + d['AT'] = result.address + } + } + } + + },0) + } + + def amendMatch(result, address){ + def matchResult = result.matchResult + if(result.credit > 0.65 && matchResult.distance < 2500){ + matchResult.match = true + }else if(result.credit>0.5 && address && result.address && findAddress(result.address)){ + if(result.address.contains(address) || address.contains(result.address)){ + matchResult.match = true + } + }else if(address && address.contains(result.district) && address.contains(result.name)){ + matchResult.match = true + } + return matchResult + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/EvaluationPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/EvaluationPrice.groovy new file mode 100644 index 000000000..90eaa4e13 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/EvaluationPrice.groovy @@ -0,0 +1,47 @@ +package com.fangjia.script.gj.chongqingblank + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient +import net.sf.json.JSONObject + +class EvaluationPrice { + + static main(args) { + def task = new EvaluationPrice() + task.run() + } + + def run(){ + ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\估值3-V3.xlsx",{ d-> + if(d['K'] && d['M']){ + def priceMap = getEvaluationPrice(d['J'], d['K'], d['L'], d['D'].toString()) + if(priceMap){ + d['E'] = priceMap.avgPrice + d['F'] = priceMap.totalPrice + } + } + },0); + } + + + def getEvaluationPrice(city, region, name, area){ + Map params = new HashMap(); + params.put("city", city); + params.put("district", region); + params.put("name", name); + params.put("size", area); + params.put("__refresh_cache","1") + + JSONObject result = OpenApiClient.evaluate(params); + if (result.getInt("code") == 200) { + if (result.containsKey("result")) { + return result.getJSONObject("result") + } + } else if (result.getInt("code") == 500) { + println("小区估价失败:${params.city} ${params.name} ${params.size} ${result}") + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/TestMatchResult.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/TestMatchResult.groovy new file mode 100644 index 000000000..9ff1149d5 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/TestMatchResult.groovy @@ -0,0 +1,629 @@ +package com.fangjia.script.gj.chongqingblank + +import com.fangjia.core.RegionData +import com.fangjia.datastore.driver.DSCollection +import com.fangjia.datastore.driver.DataStore +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.gj.district.NameProcess +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.GeoHelper +import com.fangjia.script.util.OpenApiClient +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import com.fangjia.script.yinjihaun.amap.GaoDeMapApiData +import com.mongodb.BasicDBObject +import com.mongodb.DBObject + +class TestMatchResult { + + private static final DSCollection SEAWEED = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed") + + static main(args) { + def task = new TestMatchResult() +// task.run1("D:\\数据\\重庆农商行\\估值1.xlsx") + task.run2("D:\\数据\\重庆农商行\\估值2-V2.xlsx") +// task.run3("D:\\数据\\重庆农商行\\估值3.xlsx") + +// task.nearDistRun("D:\\数据\\重庆农商行\\估值2-V5.xlsx") +// task.run2V1("D:\\数据\\重庆农商行\\估值2-V2.xlsx") + +// task.similarScore("D:\\数据\\重庆农商行\\估值2-V5.xlsx") + +// task.mapRun("D:\\数据\\重庆农商行\\估值2-V4-map.xlsx"); + + } + + + + def run1(def fileName){ + ExcelUtils.multiThreadingModify(fileName, { d -> + def region = d['I'] + def city = d['J'] + if (!d['I'] && !d['J']) { + city = d['B'] + region = d['C'] + } + def newName = null + if(d['D']=="大川菲尔小城"){ + if (d['D']) { + newName = d['D'].toString().replace(".", "").replace("(", "").replace(")", "").replace("。", "") + .replace("·", "").replace("-", "").replace("`", "").replace("?", "").replace("‘", "") + .replace("1期", "一期").replace("2期", "二期").replace("3期", "三期") + .replace("4期", "四期").replace("5期", "五期").replace("12区", "十二区").replace("1区", "一区").replace("2区", "二区").replace("3区", "三区") + .replace("4区", "四区").replace("5区", "五区").replace("a区", "A区") + .replace("B区", "B区").replace("c区", "C区").replace("d区", "D区") + } + if (!region || (region && region == d['L'])) { + if (d['D'] && !d['O'] && d['M']) { + if (d['M'] == newName) { + d['O'] = "相同" + } + if (!d['O']) { + def dbo = CommonDatautils.findDistrictByAlias(d['K'], d['L'], d['D'], "district") + if (dbo) { + d['O'] = "别名匹配" + } + } +// if (!d['O'] && d['M'] && d['E']) { +// if (d['E'].toString().contains(d['M'])) { +// d['O'] = "名称in地址" +// } +// } + + if (!d['O']) { + def py1 = com.fangjia.core.util.DataUtil.hanyuQuanpin(newName, " ") + def py2 = com.fangjia.core.util.DataUtil.hanyuQuanpin(d['M'], " ") + if (py1 == py2) { + d['O'] = "拼音相同" + } + } + + if (!d['O'] && d['D'] && d['M'] && newName) { + def word = null + if (newName.toString().length() >= d['M'].toString().length()) { + word = newName.toString().replace(d['M'], "") + } else if (newName.toString().length() < d['M'].toString().length()) { + word = d['M'].toString().replace(newName, "") + } + if (word) { + def words = ["重庆", "小区", "(高层)", "花园", "恒都", "(洋房)", "社区"] + for (def w : words) { + if (word == w) { + d['O'] = "去部分词后相同" + break + } + } + + if (!d['O'] && !word.endsWith("区") && !word.endsWith("期") && !word.endsWith("团") && !word.endsWith("东苑") && !word.endsWith("西苑") && !word.endsWith("南苑") && !word.endsWith("北苑")) { + if (d['N'] && d['N'].toString().endsWith("号") && d['E'] && d['E'].toString().contains(d['N'])) { + + def score = CosineSimilarAlgorithm.getSimilarity(d['M'], newName); + if (score >= 0.65) { + println newName + " " + d['M'] + " " + score + d['O'] = "地址相同" + } + } + } + } + } + + if (!d['O'] && newName && !newName.contains("期") && !newName.endsWith("组团") && (!newName.endsWith("区") || newName.endsWith("小区"))) { + + def gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, d['E'], "住宅", "高德") + if (!gaodeRest) { + gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, d['E'], "住宅", "百度") + } + if (gaodeRest && gaodeRest.name) { + def mapName = gaodeRest.name.replace(".", "").replace("(", "").replace(")", "").replace("·", "").replace("`", "").replace("?", "") + if (mapName == d['M'] && gaodeRest.score >= 0.7) { + d['O'] = gaodeRest.source + "匹配" + println newName + " " + d['M'] + " " + gaodeRest.score + } + } + } + } + } + + + if (d['D'] && !d['O']) { + def gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, d['E'], "住宅", "高德") + def mapName = null + if (gaodeRest && gaodeRest.name) { + mapName = gaodeRest.name.replace(".", "").replace("(", "").replace(")", "").replace("·", "").replace("`", "").replace("?", "") + } + if (!gaodeRest || (gaodeRest && mapName != newName)) { + gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, d['E'], "住宅", "百度") + if (gaodeRest && gaodeRest.name) { + mapName = gaodeRest.name.replace(".", "").replace("(", "").replace(")", "").replace("·", "").replace("`", "").replace("?", "") + } + } + if (gaodeRest && mapName == newName) { + d['P'] = gaodeRest.district + d['Q'] = mapName + d['R'] = gaodeRest.address + d['S'] = gaodeRest.lng + d['T'] = gaodeRest.lat + d['U'] = gaodeRest.type + d['v'] = gaodeRest.source + } + } + } + + },0); + } + + def similarScore(def fileName){ + ExcelUtils.multiThreadingModify(fileName, { d -> + if(d['D'] && d['M']){ + def score = CosineSimilarAlgorithm.getSimilarity(d['D'], d['M']); + d['Q'] = score + } + },0) + } + + + def run2(def fileName){ + ExcelUtils.multiThreadingModify(fileName,{ d-> + def region = d['I'] + def city = d['J'] + if(!d['I']){ + city = d['J'] + } + +// if(d['E']=="巴南区花溪镇花溪新街88号2单元3-4#") { +// println "ss" + + + if (!region || (region && region == d['L'])) { + if (d['N'] && !d['O'] && d['E']) { + if (d['N'].toString().endsWith("号") && d['E'] && d['E'].toString().contains(d['N'])) { + d['O'] = "地址匹配" + } + } + } + + if (!d['O'] && d['M'] && d['E']) { + def matchName = d['M'] + if(matchName.toString().endsWith("小区")){ + matchName = matchName.toString().replace("小区","") + } + if(matchName.toString().startsWith(d['L'])){ + matchName = matchName.toString().replace(d['L'],"") + } + def oldName= NameProcess.nameStandard(city,region, d['E']) + if (oldName.toString().contains(matchName)) { + d['O'] = "名称in地址" + } + } + + if (d['E'] && !d['O'] ) { + def gaodeRest = null + + if (!gaodeRest && d['E']) { + gaodeRest = FangjiaClientUtils.getMapData(city, region, d['E'], null, "住宅", "高德") + } + if(!gaodeRest && d['H']){ + gaodeRest = FangjiaClientUtils.getMapData(city, region, d['H'], null, "住宅", "高德") + } + if (!gaodeRest && d['E']) { + gaodeRest = FangjiaClientUtils.getMapData(city, region, d['E'], null, "住宅", "百度") + } + if (!gaodeRest && d['H']) { + gaodeRest = FangjiaClientUtils.getMapData(city, region, d['H'], null, "住宅", "百度") + } + + if (gaodeRest && gaodeRest.score >= 0.5) { + d['P'] = gaodeRest.district + d['Q'] = gaodeRest.name + d['R'] = gaodeRest.address + d['S'] = gaodeRest.lng + d['T'] = gaodeRest.lat + d['U'] = gaodeRest.type + d['v'] = gaodeRest.source + } + } + + if(!d['O']){ + def locationMap = getLocation(city, d['E']) + if(locationMap){ + def mapRets = findNearestByMap(locationMap.lat, locationMap.lng) + if(mapRets){ + def o = mapRets.get(0) + d['W'] = o.cityname + d['X'] = o.adname + d['Y'] = o.name + if(o.address && o.address instanceof String){ + d['Z'] = o.address + } + d['AA'] = o.type + d['AB'] = o.distance + d['AC'] = o.location + } + def dbRets = findNearestDistrict(city, locationMap.lat, locationMap.lng, 1000, 10) + if(dbRets){ + def b = dbRets.get(0) + d['AD'] = b.region + d['AE'] = b.name + d['AF'] = b.address + d['AG'] = b.distance + } + } + } + +// } + + },0); + } + + def distanceRun(def fileName){ + ExcelUtils.multiThreadingModify(fileName, { d -> + def city = d['J'] + if(!d['O'] && d['E'] && d['E'].toString().length()>6){ + def locationMap = getLocation(city, d['E']) + if(locationMap){ + def mapRets = findNearestByMap(locationMap.lat, locationMap.lng) + if(mapRets){ + def o = mapRets.get(0) + d['W'] = o.cityname + d['X'] = o.adname + d['Y'] = o.name + if(o.address && o.address instanceof String){ + d['Z'] = o.address + } + d['AA'] = o.type + d['AB'] = o.distance + d['AC'] = o.location + } + def dbRets = findNearestDistrict(city, locationMap.lat, locationMap.lng, 1000, 10) + if(dbRets){ + def b = dbRets.get(0) + d['AD'] = b.region + d['AE'] = b.name + d['AF'] = b.address + d['AG'] = b.distance + } + } + } + },0); + } + + def nearDistRun(def fileName){ + ExcelUtils.multiThreadingModify(fileName, { d -> + def city = d['B'] +// if(!d['I'] && !d['J']){ +// city = d['B'] +// } + if(d['O'] && d['O']=="周边"){ + def locationMap = null + if(d['B'] && d['E']){ + locationMap = getLocation(city, d['E']) + } +// if(!locationMap && d['C']){ +// locationMap = getLocation(city, d['C']) +// } + + if(locationMap){ + def dbRets = findNearestDistrict(city, locationMap.lat, locationMap.lng, 1000, 10) + if(dbRets){ + def b = dbRets.get(0) + d['K'] = b.city + d['L'] = b.region + d['M'] = b.name + d['N'] = b.address + d['P'] = b.distance + } + } + } + },0); + } + + def run2V1(def fileName){ + ExcelUtils.multiThreadingModify(fileName,{ d-> + def city = d['J'] + if(d['O']){ + d['AH'] = d['L'] + d['AI'] = d['M'] + d['AJ'] = d['N'] + } + if(!d['O']){ + if(d['R'] && d['R'].contains("号")){ + def addr = d['R'].toString().substring(0, d['R'].indexOf("号")) + if((addr && d['E'] && d['E'].contains(addr))){ + def region = RegionData.formatRegion(city, d['P']) + if(d['U'] && d['U'].toString().contains("住宅")){ + d['AH'] = region + d['AI'] = d['Q'] + d['AJ'] = d['R'] + d['AK'] = d['U'] + d['AM'] = d['S']+","+d['T'] + d['O'] ="地图匹配" + }else if(!d['U']){ + d['AH'] = region + d['AI'] = d['Q'] + d['AJ'] = d['R'] + d['AK'] = d['U'] + d['AM'] = d['S']+","+d['T'] + d['O'] ="地图匹配" + } + } + } + } + + if(!d['O']){ + if(d['Y'] && d['AE']){ + def mapName = d['Y'].toString().replace(".", "").replace("(", "").replace(")", "").replace("。", "") + .replace("·", "").replace("-", "").replace("`", "").replace("?", "").replace("‘", "") + .replace("1期", "一期").replace("2期", "二期").replace("3期", "三期") + .replace("4期", "四期").replace("5期", "五期").replace("12区", "十二区").replace("1区", "一区").replace("2区", "二区").replace("3区", "三区") + .replace("4区", "四区").replace("5区", "五区").replace("a区", "A区") + .replace("B区", "B区").replace("c区", "C区").replace("d区", "D区") + if(mapName == d['AE']){ + def region = RegionData.formatRegion(city, d['AD']) + d['AH'] = region + d['AI'] = d['AE'] + d['AJ'] = d['AF'] + d['AK'] = d['DB'] + d['O'] ="名称相同" + }else{ + if(d['AB'].toString().toDouble()>=d['AG'].toString().toDouble()){ + def region = RegionData.formatRegion(city, d['AD']) + d['AH'] = region + d['AI'] = d['AE'] + d['AJ'] = d['AF'] + d['AK'] = "DB库近" + d['AL'] = d['AG'] + d['O'] ="距离比较" + }else{ + def region = RegionData.formatRegion(city, d['X']) + d['AH'] = region + d['AI'] = d['Y'] + d['AJ'] = d['Z'] + d['AK'] = d['AA'] + d['AL'] = d['AB'] + d['AM'] = d['AC'] + d['O'] ="距离比较" + } + } + }else if(d['Y']){ + def region = RegionData.formatRegion(city, d['X']) + d['AH'] = region + d['AI'] = d['Y'] + d['AJ'] = d['Z'] + d['AK'] = d['AA'] + d['AL'] = d['AB'] + d['AM'] = d['AC'] + d['O'] ="采用地图相近" + }else if(d['AE']){ + def region = RegionData.formatRegion(city, d['AD']) + d['AH'] = region + d['AI'] = d['AE'] + d['AJ'] = d['AF'] + d['AK'] = "DB库近" + d['AL'] = d['AG'] + d['O'] ="采用DB相近" + } + } + },0); + } + + + def findNearestByMap(lng, lat){ + def rets = GaoDeMapApiData.doSearchByGaoDe("120000|120203|120300|120301|120302|120303|120304", lng, lat, 1000, 1, 10) + return rets + + } + + public List findNearestDistrict(def city, double lat, double lng, int radiusMeters, int limit) { + DBObject districtFields = new BasicDBObject("city", 1).append("region", 1).append("name", 1).append("address", 1) + districtFields.put("estate_type2", 1) + districtFields.put("lat", 1) + districtFields.put("lng", 1) + + + DBObject sdq = new BasicDBObject(); + sdq.put("sd.point", lat + "," + lng); + sdq.put("sd.sfield", "lat_lng"); + sdq.put("sd.distance", String.valueOf((double) radiusMeters / 1000.0)); + + DBObject q = new BasicDBObject(); + q.put("cat", "district"); + q.put("city", city); + q.put("visible", 0); + q.put('$sd', sdq); + + List list = new ArrayList() + Iterator iter = SEAWEED.find(q, districtFields).limit(limit * 2).iterator() + while(iter.hasNext()) { + def o = iter.next() + o.distance = GeoHelper.GetDistance(o.lat/10000000000.0, o.lng/10000000000.0, lat, lng) + list.add(o) + } + + if (list.size() == 0) { + return Collections.EMPTY_LIST; + } + list.sort{o1,o2-> + return o1.distance <=> o2.distance + } + if (list.size() > limit) { + list = list.subList(0, limit); + } + return list; + } + + def getLocation(city, addr){ + + if(addr){ + def rMap = [:] + def reps = GaoDeApiUtils.geo(city, addr); + if(reps && reps.count.toInteger()>=1){ + def map = reps.geocodes.get(0) + def location = map.location + def level = map.level + if(location){ + def locMap = location.split(",") +// def lat = dist.lat/10000000000.0; +// def lng = dist.lng/10000000000.0; +// def distance = GeoHelper.GetDistance(locMap[1].toDouble(), locMap[0].toDouble(), lat, lng) +// rMap.distance = distance +// rMap.level = level + rMap.lat = locMap[1].toDouble() + rMap.lng = locMap[0].toDouble() +// rMap.keyword = map.formatted_address +// rMap.location = location +// rMap.region = map.district + return rMap; + } + } + } + return null; + } + + + + def run3(def fileName){ + ExcelUtils.multiThreadingModify(fileName,{ d-> + def city = d['I'] + def region = d['J'] + + def newName = null + if(d['C']){ + if(d['C'].toString().length()<2){ + newName = null + }else{ + newName = d['C'].toString().replace(".", "").replace("(", "").replace(")", "").replace("。", "") + .replace("·", "").replace("-", "").replace("`", "").replace("?", "").replace("‘", "") + .replace("1期", "一期").replace("2期", "二期").replace("3期", "三期") + .replace("4期", "四期").replace("5期", "五期").replace("12区", "十二区").replace("a区", "A区") + .replace("B区", "B区").replace("c区", "C区").replace("d区", "D区") + } + + } + def address = null + if(d['H']){ + address = d['H'] + }else{ + def arrs = d['B'].split("->") + if(arrs && arrs.size()==4){ + address = arrs[3] + }else if(arrs && arrs.size()==2){ + address = arrs[1] + } + } + if(!region ||(region && d['L'] && region.contains(d['L']))) { + if (d['B'] && !d['O'] && d['M']) { + if (d['M'] == newName) { + d['O'] = "相同" + } + if (!d['O']) { + def dbo = CommonDatautils.findDistrictByAlias(d['K'], d['L'], newName, "district") + if (dbo) { + d['O'] = "别名匹配" + } + } + + if (!d['O']) { + def py1 = com.fangjia.core.util.DataUtil.hanyuQuanpin(newName, " ") + def py2 = com.fangjia.core.util.DataUtil.hanyuQuanpin(d['M'], " ") + if (py1 == py2) { + d['O'] = "拼音相同" + } + } + + if (!d['O'] && d['M'] && newName) { + def word = null + if (newName.toString().length() >= d['M'].toString().length()) { + word = newName.toString().replace(d['M'], "") + } else if (newName.toString().length() < d['M'].toString().length()) { + word = d['M'].toString().replace(newName, "") + } + if (word) { + def words = ["重庆", "小区", "(高层)", "花园", "恒都", "(洋房)", "社区"] + for (def w : words) { + if (word == w) { + d['O'] = "去部分词后相同" + break + } + } + + if (!d['O'] && !word.endsWith("区") && !word.endsWith("期") && !word.endsWith("团") && !word.endsWith("东苑") && !word.endsWith("西苑") && !word.endsWith("南苑") && !word.endsWith("北苑")) { + if (d['N'] && d['N'].toString().endsWith("号") && address.toString().contains(d['N'])) { + + def score = CosineSimilarAlgorithm.getSimilarity(d['M'], newName); + if (score >= 0.65) { + println newName + " " + d['M'] + " " + score + d['O'] = "地址相同" + } + } + } + } + } + + if (!d['O'] && newName && !newName.contains("期") && !newName.endsWith("组团") && (!newName.endsWith("区") || newName.endsWith("小区"))) { + + def gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, address, "住宅", "高德") + if (!gaodeRest) { + gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, address, "住宅", "百度") + } + if (gaodeRest && gaodeRest.name) { + def mapName = gaodeRest.name.replace(".", "").replace("(", "").replace(")", "").replace("·", "").replace("`", "").replace("?", "") + if (mapName == d['M'] && gaodeRest.score >= 0.7) { + d['O'] = gaodeRest.source + "匹配" + println newName + " " + d['M'] + " " + gaodeRest.score + } + } + } + } + } + + + + if(newName && !d['O']){ + def gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, address, "住宅", "高德") + def mapName = null + if(gaodeRest && gaodeRest.name){ + mapName = gaodeRest.name.replace(".", "").replace("(", "").replace(")", "").replace("·", "").replace("`", "").replace("?", "") + } + if(!gaodeRest || (gaodeRest && mapName != newName)) { + gaodeRest = FangjiaClientUtils.getMapData(city, region, newName, d['E'], "住宅", "百度") + if(gaodeRest && gaodeRest.name){ + mapName = gaodeRest.name.replace(".", "").replace("(", "").replace(")", "").replace("·", "").replace("`", "").replace("?", "") + } + } + if(gaodeRest && mapName== newName){ + d['P'] = gaodeRest.district + d['Q'] = mapName + d['R'] = gaodeRest.address + d['S'] = gaodeRest.lng + d['T'] = gaodeRest.lat + d['U'] = gaodeRest.type + d['v'] = gaodeRest.source + } + } + + },0); + } + + def mapRun(fileName){ + ExcelUtils.multiThreadingModify(fileName, { d -> + def city = d['J'] + def region = d['I'] + if(city && region && d['Q'] && d['O']!="距离比较" && d['O']!="地址匹配"){ + def gaodeRest = FangjiaClientUtils.getMapData(city, region, d['Q'], null, "住宅", "高德") + if(gaodeRest){ + d['V'] = gaodeRest.city + d['W'] = gaodeRest.district + d['X'] = gaodeRest.name + d['Y'] = gaodeRest.address + d['Z'] = gaodeRest.lng + d['AA'] = gaodeRest.lat + d['AB'] = gaodeRest.type + } + + d['AC'] = NameProcess.nameStandard(city,region, d['Q']) + } + },0) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/TestPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/TestPrice.groovy new file mode 100644 index 000000000..e252654d5 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/TestPrice.groovy @@ -0,0 +1,89 @@ +package com.fangjia.script.gj.chongqingblank + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.QueryOperators + +class TestPrice { + + def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; + public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) + + static main(args) { + def task = new TestPrice() + task.run() + } + + def run(){ + ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\估值1-V2.xlsx",{ d-> + if(d['K'] && d['M']){ + def dbo = findThirdPrice(d['K'], d['L'], d['M'], 202012, "lianjia_esf_price") + if(dbo){ + d['AU'] = dbo.lianjia_esf_price + } + + def district = CommonDatautils.findDistrict(d['K'], d['L'], d['M'], "district") + def startDate = 20201101 + def endDate = 20210601 + def trades = findTrades(district, startDate, endDate) + if(trades){ + d['AV'] = (int)(trades.sum()/trades.size()) + d['AW'] = trades.size() + } + } + },0); + } + + def findThirdPrice(def city, def region, def name, def s_date,def fieldName){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("name", name) + q.put("s_date", new BasicDBObject(QueryOperators.GTE, s_date)) + q.put(fieldName, new BasicDBObject(QueryOperators.EXISTS, true)) + def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date",-1)) + def obj = null; + for(def d : dbcursor){ + obj = d; + break; + } + return obj; + } + + def findTrades(dist, startDate, endDate){ + def q = new BasicDBObject() + q.put("city", dist.city) + q.put("region", dist.region) +// q.put("source", "链家在线") + + def alias = [] + if(dist.alias){ + if(!dist.alias.contains(dist.name)){ + dist.alias << dist.name + } + alias = dist.alias + }else{ + alias << dist.name + } + + if(alias){ + q.put("district_name", new BasicDBObject('$in', alias)) + } + + q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate)) + def dbcorsor = DEAL_PRICE_COLL.find(q) + def prices = [] + for(def d : dbcorsor){ + if(d.avg_price){ + prices << d.avg_price + } + } + + return prices + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/FieldStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/FieldStat.groovy index 72fd50274..eb7d4aadd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/FieldStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/FieldStat.groovy @@ -1,39 +1,106 @@ package com.fangjia.script.gj.data import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection class FieldStat { + final static DBCollection CITY_INFO_CONN = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", MongoUtils.MongoDB_Domain_mongo); + static main(args) { def task = new FieldStat(); - task.run() +// task.run() + task.run1() +// task.run2() + } + + def run1(){ + def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] + def dist_total = 0 + + def addr_alias_total = 0 + def rets = [] + rets << ["城市","别名总量","有别名小区数量","小区总量","有别名小区占比"] + for(def city : citys){ + def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias", "addr_alias"]); + def alias_index = 0 + def alias_total = 0 + + for(def d : districts){ + def alias = d.alias + if(alias){ + alias_index++ + alias_total = alias_total + alias.size() + } + } + def precent = DataHandleUtils.getDoubleRound(2, alias_index*100.0/districts.size()) + rets << [city, alias_total, alias_index, districts.size(), precent+"%"] + + } + ExcelUtils.write("d:\\数据\\92个城市别名统计.xlsx", rets, "统计") +// println dist_total +// println alias_total +// println addr_alias_total } def run(){ - ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\建行数据融合结果统计_0425.xlsx", { row -> - def city = row['A'] + ExcelUtils.multiThreadingModify("D:\\数据\\城市统计.xlsx", { row -> + def city = row['B'] if(city){ - def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias"]); + def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); def total = districts.size() + row['D'] = total - def aliasIndex = 0 - def aliasTotalCount = 0 - for(def d : districts){ - def alias = d.alias - if(alias){ - aliasIndex++ - aliasTotalCount = aliasTotalCount + alias.size() - } - } -// row['J'] = aliasTotalCount -// row['K'] = aliasIndex - def p = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, aliasIndex*100/total) - row['B'] = p+"%" -// row['M'] = total - println city +" "+p +// def aliasIndex = 0 +// def aliasTotalCount = 0 +// for(def d : districts){ +// def alias = d.alias +// if(alias){ +// aliasIndex++ +// aliasTotalCount = aliasTotalCount + alias.size() +// } +// } +//// row['J'] = aliasTotalCount +//// row['K'] = aliasIndex +// def p = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, aliasIndex*100/total) +// row['B'] = p+"%" +//// row['M'] = total +// println city +" "+p } - }, 1); + }, 0); println "finish" } + + def run2(){ + def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] + def rets = [] + rets << ["城市","建筑年代覆盖率","物业类型覆盖率","建筑类型覆盖率","区域最高小区均价覆盖率","区域最低小区均价覆盖率"] + for(def c : citys){ + def obj = findCoverPrecent(c) + if(obj){ + def datePrecent = DataHandleUtils.getDoubleRound(2, obj.district_detail.finishBuildingDate*100) + def estateTypePrecent = DataHandleUtils.getDoubleRound(2, obj.district_detail.estateType2*100) + def buildTypePrecent = DataHandleUtils.getDoubleRound(2, obj.district_detail.buildType*100) + rets << [c, datePrecent+"%", estateTypePrecent+"%", buildTypePrecent+"%", "100%", "100%"] + } + } + + ExcelUtils.write("d:\\数据\\92个城市覆盖率统计.xlsx", rets, "统计") + + } + + def findCoverPrecent(def city){ + def q = new BasicDBObject() + q.put("city", city) + def dbcursor = CITY_INFO_CONN.find(q).sort(new BasicDBObject("weekend", -1)).limit(1) + def obj = null + for(def d : dbcursor){ + obj = d + } + return obj + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/RegionExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/RegionExport.groovy new file mode 100644 index 000000000..6e967d2bc --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/RegionExport.groovy @@ -0,0 +1,29 @@ +package com.fangjia.script.gj.data + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.ExcelUtils + +class RegionExport { + + static main(args) { + def task = new RegionExport() + task.run() + } + + def run(){ + ExcelUtils.multiThreadingModify("d:\\数据\\亲邻\\小区城市列表.xls",{d-> + def city = d['B'] + def newCity = RegionData.formatCity(city) + if(newCity){ + def regions = CommonUtils.getAllRegions(newCity, "region") + def regionStr = "" + regions.each{k,v-> + regionStr = regionStr + k+"\n" + } + d['C'] = regionStr + } + },0) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ExportData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ExportData.groovy index fd2856af7..6a44d4139 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ExportData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ExportData.groovy @@ -24,8 +24,8 @@ class ExportData { static main(args) { def task = new ExportData(); -// task.exportThird() - task.exportMap(); + task.exportThird() +// task.exportMap(); } def exportMap(){ @@ -71,7 +71,7 @@ class ExportData { def exportThird(){ - def citys = ["淮安","昆山","连云港","南京","南通","宿迁","苏州","泰州","无锡","徐州","盐城","扬州","镇江"] + def citys = ["重庆"] for(def city : citys){ def rets = [] rets << ["城市","区域","名称","别名"] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index e0f5df506..a131a414c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -3,13 +3,18 @@ package com.fangjia.script.gj.district import com.fangjia.core.util.HttpRequest import net.sf.json.JSONArray import net.sf.json.JSONObject +import org.apache.commons.lang.StringUtils class FangjiaClientUtils { public static def getMapData(def city, def region, def name, def address, def types, def mapCode){ JSONObject jsonObj = null try { - def url = "http://192.168.0.19:38080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&address="+address+"&types="+types+"&mapCode="+mapCode + def url = "http://192.168.0.19:38080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&types="+types+"&mapCode="+mapCode + //def url = "http://192.168.10.166:8080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&types="+types+"&mapCode="+mapCode + if(address){ + url = url + "&address="+address + } // def url = "http://localhost:8080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&address="+address+"&types="+types+"&mapCode="+mapCode def content = new HttpRequest().newGet(url, null, "utf-8") jsonObj = JSONObject.fromObject(content) @@ -44,4 +49,34 @@ class FangjiaClientUtils { println content } + + + public static def searchMapData(def city, def keyword, def types, def mapCode){ + def jsonObj = null + try { + def url = "http://192.168.0.19:38080/FangJiaDataService/search?city="+city+"&keyword="+keyword+"&types="+types+"&mapCode="+mapCode+"&limit=10" + def content = new HttpRequest().newGet(url, null, "utf-8") + // def ret = content.replace("[","").replace("]","") + jsonObj = JSONArray.fromObject(content) + } catch (Exception e) { + return null; + } + return jsonObj; + } + + + public static Map JsonToMap(JSONObject json) { + Map columnValMap = new HashMap(); + Set jsonKeys = json.keySet(); + for (Object key : jsonKeys) { + Object JsonValObj = json.get(key); + if (key instanceof JSONObject) { + columnValMap.put((String) key, JsonToMap((JSONObject) JsonValObj)); + } else { + columnValMap.put((String) key, JsonValObj); + } + } + return columnValMap; + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy new file mode 100644 index 000000000..7860afad9 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy @@ -0,0 +1,236 @@ +package com.fangjia.script.gj.es + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.apache.lucene.queryparser.classic.QueryParser + +class EsMonthHouseStat { + + public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; + public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) + + static main(args) { + def task = new EsMonthHouseStat() +// task.run() + task.tradeRun() +// task.houseRun() + } + + def run(){ + def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] + def rets = [] + def dates = ["2020-05","2020-06","2020-07","2020-08","2020-09","2020-10","2020-11","2020-12","2021-01","2021-02","2021-03","2021-04","2021-05"] + def headers = ["城市"] + headers.addAll(dates) + rets << headers + for(def city : citys){ + def datas = [] + datas << city + for(def d : dates){ + def date = DateUtils.parseFromTightYYYYMMDD(d+"-01") + def startDate = DateUtils.formatDateByPattern(date, "yyyyMMdd") + def endDate = DateUtils.addMonth(date, 1).replace("-","") + def count = findHouseCount(city, startDate.toInteger(), endDate.toInteger()) + datas << count + } + rets << datas + println datas + } + ExcelUtils.write("d:\\数据\\建行92个城市房源统计.xlsx", rets, "统计") + } + + def tradeRun(){ + def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] + def rets = [] + def rets2 = [] + //, + //"2021-01","2021-02","2021-03","2021-04","2021-05" + //def dates = ["2020-01","2020-02","2020-03","2020-04","2020-05","2020-06","2020-07","2020-08","2020-09","2020-10","2020-11","2020-12"] + def dates = ["2021-04"] + def headers = ["城市"] + headers.addAll(dates) + rets << headers + + + def headers2 = ["城市"] + headers2.addAll(dates) + rets2 << headers2 + for(def city : citys){ + def datas = [] + datas << city + + def datas2 = [] + datas2 << city + for(def d : dates){ + def startDate = DateUtils.parseFromTightYYYYMMDD(d+"-01") + def endDateStr = DateUtils.addMonth(startDate, 1).replace("-","") + def endDate = DateUtils.parseFromTightYYYYMMDD2(endDateStr) + //def count = findTrades(city, startDate, endDate) + def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias"]); + def oldCount = 0 + def newCount = 0 + for(def dist : districts){ + def countMap = findTradesDetail(dist, startDate, endDate) + oldCount = oldCount + countMap.total_old + newCount = newCount + countMap.total_new + println dist.city + " " + dist.name +" "+countMap.total_old+" "+countMap.total_new + } + + datas << oldCount + datas2 << newCount + println city +" "+ oldCount +" "+newCount + } + rets << datas + + rets2 << datas2 + println datas + } + ExcelUtils.write("d:\\数据\\建行92个城市成交统计(去重前匹配).xlsx", rets, "统计") + + ExcelUtils.write("d:\\数据\\建行92个城市成交统计(去重后匹配).xlsx", rets2, "统计") + } + + def houseRun(){ + ExcelUtils.multiThreadingModify("d:\\数据\\小区案例数据覆盖率.xlsx",{d-> + def city = d['B'] + if(city == "湖州"){ + def count = findHouseCount(city, 20210301, 20210601) + d['E'] = count + + def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias"]); + def disthouseCount = 0 + for(def dist : districts){ + def alias = [] + if(dist.alias){ + if(!dist.alias.contains(dist.name)){ + dist.alias << dist.name + } + alias = dist.alias + }else{ + alias << dist.name + } + + def distCount = findHouseCount(dist.city, dist.region, alias, 20210301, 20210601) + if(distCount){ + disthouseCount++ + } + println dist.city+" "+dist.region+" "+dist.name+" "+distCount + } + + println "disthouseCount:"+disthouseCount + println districts.size() + def precent = DataHandleUtils.getDoubleRound(2, disthouseCount*100.0/districts.size()) + d['D'] = precent+"%" + } + },0) + } + + + def findHouseCount(def city, def region, def alias, def startDate, def endDate){ + def aliasStr = null; + def total = 0 + try{ + aliasStr = escapeExprSpecialWord(alias.join("','")) + String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}')" + + ElasticUtils.executeSqlQuery(sql,{data-> + total = data.c + }) + }catch(Exception e){ + e.printStackTrace() + } + + return total + } + + /** + * 转义字符串中的特殊字符 + * 仅过滤Lucene regexps, 其他的勿用 + * @param keyword + * @return + */ + public static String escapeExprSpecialWord(String keyword) { + return QueryParser.escape(keyword); + } + + def findHouseCount(def city, def startDate, def endDate){ + + String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} " + def total = 0 + ElasticUtils.executeSqlQuery(sql,{ data-> + total = data.c + }) + return total + } + + + + + + def findTrades(city, startDate, endDate){ + def q = new BasicDBObject() + q.put("city", city) +// q.put("region", dist.region) +// q.put("source", "链家在线") + +// def alias = [] +// if(dist.alias){ +// if(!dist.alias.contains(dist.name)){ +// dist.alias << dist.name +// } +// alias = dist.alias +// }else{ +// alias << dist.name +// } + +// if(alias){ +// q.put("district_name", new BasicDBObject('$in', alias)) +// } + + q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate)) + def count = DEAL_PRICE_COLL.count(q) + return count + } + + + + def findTradesDetail(dist, startDate, endDate){ + def q = new BasicDBObject() + q.put("city", dist.city) + q.put("region", dist.region) + + def alias = [] + if(dist.alias){ + if(!dist.alias.contains(dist.name)){ + dist.alias << dist.name + } + alias = dist.alias + }else{ + alias << dist.name + } + + if(alias){ + q.put("district_name", new BasicDBObject('$in', alias)) + } + q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate)) + def dbcursor = DEAL_PRICE_COLL.find(q) + def total = dbcursor.count() + def rets = new HashSet() + for(def d : dbcursor){ + def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd") + if(!rets.contains(key)){ + rets << key + } + } + def rMap = [:] + rMap.total_old = total + rMap.total_new = rets.size() + return rMap + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsSelectTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsSelectTest.groovy new file mode 100644 index 000000000..6f2307343 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsSelectTest.groovy @@ -0,0 +1,66 @@ +package com.fangjia.script.gj.es + +import com.fangjia.core.util.DateUtil +import com.fangjia.script.util.ElasticUtils +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders +import org.elasticsearch.xpack.sql.jdbc.EsDataSource + +import java.sql.Connection +import java.sql.ResultSet +import java.sql.Statement + +class EsSelectTest { + EsDataSource dataSource = new EsDataSource(); + String address = "jdbc:es://http://192.168.0.86:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"; + + static main(args) { + def task = new EsSelectTest() + task.run() + } + + def run(){ + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("request", "/ws/czbank")) + //.must(QueryBuilders.termQuery("request_body", "findSample")) + + + def houses = ElasticUtils.executeSimpleQuery("192.168.0.86", "fangjia-log*", queryBuilder); + if(houses) { + for (int k = 0; k < houses.size(); k++) { + println k + } + } + + + def endDate = new Date() + dataSource.setUrl(address) + Properties connectionProperties = new Properties() + dataSource.setProperties(connectionProperties) + StringBuffer sqlBuffer = new StringBuffer(); + String d = DateUtil.formatByPattern(new Date(), "yyyy.MM.dd") + long startD = DateUtil.getBeforeMinute(new Date(), -7).getTime(); + long endD = DateUtil.getBeforeMinute(endDate, 1).getTime(); + println(DateUtil.formatByPattern(new Date(startD), "yyyy-MM-dd HH:mm:ss")+"===="+DateUtil.formatByPattern(new Date(endD), "yyyy-MM-dd HH:mm:ss")); + + + sqlBuffer.append("select request_time FROM \"fangjia-log*\" WHERE 1=1"); + sqlBuffer.append(" and request='/ws/czbank' and request_body='findSample'") + + ResultSet rs = null + try { + Connection connection = dataSource.getConnection() + Statement statement = connection.createStatement() + rs = statement.executeQuery(sqlBuffer.toString()) + } catch (Exception e) { + e.printStackTrace() + } + StringBuffer msgBuffer = new StringBuffer(); + while (rs.next()) { + + def responsetime = rs.getLong("responsetime"); + def timestamp = DateUtil.formatByPattern(new Date(rs.getLong("time")), "yyyy-MM-dd HH:mm") + println responsetime + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy new file mode 100644 index 000000000..55e0943ab --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy @@ -0,0 +1,171 @@ +package com.fangjia.script.gj.location + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.looplineinfo.InsidePolygon +import com.fangjia.script.looplineinfo.Point +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.GeoHelper +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import java.text.SimpleDateFormat +import java.util.regex.Matcher +import java.util.regex.Pattern; + + +class LocationTest { + + static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline") + + static main(args) { + def task = new LocationTest() +// task.run() + task.run2() + } + + def run(){ + def regionPointsMap = findRegionRange() + def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region", "name", "address", "lat", "lng"]); + def index = 0 + def rets = [] + for(def dist : districts){ + def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, regionPointsMap.get(dist.region)) + if(!flag){ + index++ + + def ret = [] + ret << dist.city + ret << dist.region + ret << dist.name + + def loactionMap = matchMap(dist) + if(loactionMap){ + ret << "地图" + ret << loactionMap.region + ret << loactionMap.mapName + def distance = GeoHelper.GetDistance(dist.lat/10000000000.0, dist.lng/10000000000.0, loactionMap.lat, loactionMap.lng) + ret << distance + if(dist.region==loactionMap.region){ + println index+" "+dist.region +" "+dist.name +" 不在区域范围内,与地图相同" + }else{ + println index+" "+dist.region +" "+dist.name +" 不在区域范围内 地图:"+ loactionMap.region+" "+distance + } + }else{ + ret << "" + println index+" "+dist.region +" "+dist.name +" 不在区域范围内,地图没找到" + } + rets << ret + + } + } + ExcelUtils.write("d:\\数据\\福州\\区域经纬度错误3.xlsx", rets, "区域不在行政范围内") + } + + def run2(){ + ExcelUtils.multiThreadingModify("d:\\数据\\福州\\福州小区.xlsx") + } + + def matchMap(def dist){ + def gaodeRest = FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "高德") + def locationMap = null + if(gaodeRest){ + locationMap = getMapLocation(dist, gaodeRest); + if(locationMap){ + return locationMap + } + } + + def baiduRest = null + if (!locationMap) { + baiduRest = FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "百度") + } + if(baiduRest){ + locationMap = getMapLocation(dist, baiduRest); + } + return locationMap + } + + def getMapLocation(def dist, def rets){ + def lat = null + def lng = null + def region = null + def mapName = null + for(def r : rets){ + if(!r.city){ + continue + } + if(!r.city.contains(dist.city)){ + continue + } + def name = r.name.replace("?","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("【","").replace("】","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期").replace(" ","").trim() + if(dist.name == name){ + lat = r.lat + lng = r.lng + mapName = r.name + region = r.district + }else{ + def word = null + if (name.length() >= dist.name.length()) { + word = name.replace(dist.name, "") + } else if (name.length() < dist.name.length()) { + word = dist.name.replace(name, "") + } + if (word) { + def words = ["商住楼", "小区","公寓","别墅"] + for (def w : words) { + if (word == w) { + lat = r.lat + lng = r.lng + region = r.district + mapName = r.name + break + } + } + if(!lng){ + Pattern pattern = Pattern.compile("\\d+号楼"); + Matcher matcher = pattern.matcher(word); + if(matcher.find()){ + lat = r.lat + lng = r.lng + region = r.district + mapName = r.name + break + } + } + } + } + if(lng){ + break + } + } + if(lng){ + return ["region":region, "lng":lng, "lat":lat, "mapName":mapName] + } + + + return null + } + + def isInArea(lng, lat, points){ + List pointList = InsidePolygon.createPoint2(points); + //Point point = new Point(lng, lat); + //def flag = InsidePolygon.aboutZone(point, pointList) + def flag = InsidePolygon.isPtInPoly(lng, lat, pointList) + return flag; + } + + def findRegionRange(){ + def q = new BasicDBObject() + q.put("city","福州市") + def dbcursor = POLYLINECOLL.find(q) + def regionMap = [:] + for(def d : dbcursor){ + def region = RegionData.formatRegion("福州",d.region) + regionMap.put(region, d.polyline) + } + return regionMap + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy index 8e8bba001..b26bd079b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy @@ -4,6 +4,7 @@ import com.fangjia.core.RegionData import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject +import com.mongodb.Bytes import com.mongodb.DBCollection class CrawlDistrictIdSelect { @@ -14,8 +15,8 @@ class CrawlDistrictIdSelect { static main(args) { def task = new CrawlDistrictIdSelect() - task.run() -// task.synch() +// task.run() + task.synch() } def synch(){ @@ -50,13 +51,13 @@ class CrawlDistrictIdSelect { def findCrawlData(){ def q = new BasicDBObject() - q.put("city","福州") - q.put("source", "安居客") - q.put("s_date",new BasicDBObject('$gte', 202001)) - def dbcursor = thirdPartyColl.find(q) +// q.put("city","福州") + q.put("source", "诸葛找房") + q.put("s_date",202010) + def dbcursor = thirdPartyColl.find(q).batchSize(10000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT) def retMap = [:] - def prefix = "/community/view/" - def keyList =[] + def prefix = "xiaoqu.zhuge.com/" + def keyList =new HashSet() for(def db : dbcursor){ def url = null if(!db.url){ @@ -67,12 +68,11 @@ class CrawlDistrictIdSelect { url = db.url } if(url && url.contains(prefix) ){ - def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.size()) + def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.size()-1) if(district_id=="0"){ continue } - def shortCity = url.substring(url.indexOf("https://")+8, url.indexOf(".anjuke.com")) - def region = RegionData.formatRegion(db.city, db.region) + def shortCity = url.substring(url.indexOf("//")+2, url.indexOf(".xiaoqu.zhuge.com")) def key = shortCity +"#"+district_id def ids = retMap.get(key) @@ -83,15 +83,17 @@ class CrawlDistrictIdSelect { if(!keyList.contains(key)){ def map = new BasicDBObject() - map.root = "anjuke.com" + map.root = "zhuge.com" map.city_short= shortCity map.city = db.city - map.region = region + map.region = db.region map.district_id = district_id map.source_url = url ids << map keyList << key + }else{ + println "重复:"+key } }else{ println url +" 链接不合要求" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy index f01c92679..5a8586fb8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy @@ -32,6 +32,22 @@ class InsidePolygon { return list; } + + + public static List createPoint2(String points){ + String[] array = points.split(";"); + List list = new ArrayList(); + for(int i =0 ;i> getPoints(String points){ String[] array = points.split("-"); @@ -167,4 +183,44 @@ class InsidePolygon { return false; return true; } + + + + public static boolean isPtInPoly (double ALon , double ALat , List ps) { + int iSum, iCount, iIndex; + double dLon1 = 0, dLon2 = 0, dLat1 = 0, dLat2 = 0, dLon; + if (ps.size() < 3) { + return false; + } + iSum = 0; + iCount = ps.size(); + for (iIndex = 0; iIndex= dLat1) && (ALat < dLat2)) || ((ALat >= dLat2) && (ALat < dLat1))) { + if (Math.abs(dLat1 - dLat2) > 0) { + //得到 A点向左射线与边的交点的x坐标: + dLon = dLon1 - ((dLon1 - dLon2) * (dLat1 - ALat) ) / (dLat1 - dLat2); + // 如果交点在A点左侧(说明是做射线与 边的交点),则射线与边的全部交点数加一: + if (dLon < ALon) { + iSum++; + } + } + } + } + if ((iSum % 2) != 0) { + return true; + } + return false; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy index 476a42a3a..00ee277ea 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy @@ -1,6 +1,9 @@ package com.fangjia.script.temp.shenjian.poi import com.alibaba.fastjson.JSONObject +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher import com.fangjia.script.statistics.poi.AmapPoi import com.fangjia.script.temp.shenjian.util.GaoDeUtils import com.fangjia.script.temp.shenjian.util.HttpUtils @@ -21,7 +24,8 @@ import java.util.concurrent.Executors * @author shenjian * @since 2021/5/24 */ -class SeaweedAroundPoi { +@BatchOwner(Watcher.ShenJian) +class SeaweedAroundPoi extends BaseBatch { static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017) static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi" @@ -170,4 +174,12 @@ class SeaweedAroundPoi { put("旅游景点","110000"); put("图书馆","140500"); }}; + + @Override + void process(Object args) { + def types = listAllType() + //dealSpecialSeaweedAround(types) + dealPoi(new BasicDBObject(), types) + executorService.shutdown() + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 5adde74e3..74505e6b5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -139,12 +139,17 @@ class GaoDeUtils { ex.printStackTrace() } } else if (json && json.infocode && json.infocode == "10044") { - //throw new RuntimeException("开发者的日访问量超限,被系统自动封停,第二天0:00会自动解封") - // 多线程情况下还是exit有效果,抛异常不会停止程序 - if (keyUsed.intValue() >= GAO_DE_AK.size()) { - System.exit(1) + synchronized (this) { + for (int i = 0; i < GAO_DE_AK.size(); i++) { + if (GAO_DE_AK[i] == params.key) { + if (i + 1 > GAO_DE_AK.size()) { + System.exit(1) + } + keyUsed = new AtomicInteger(i+1) + } + } + break; } - keyUsed.incrementAndGet() } else if (json && json.infocode && json.infocode == "10004") { // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 Thread.sleep(60000) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/CrawalDBUtil.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/CrawalDBUtil.groovy index 4f6bc253e..c0cc1f468 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/CrawalDBUtil.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/CrawalDBUtil.groovy @@ -25,6 +25,7 @@ class CrawalDBUtil { static DBCollection data_process_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "data_process") static DBCollection average_statistics_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "average_statistics") static DBCollection map_district_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "map_district_process") + static DBCollection districtDetailColl = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_detail") public static getSeedsCol() { return seedsColl; @@ -67,6 +68,10 @@ class CrawalDBUtil { return map_district_col; } + public static getDistrictDetailColl(){ + return districtDetailColl; + } + static def updateSeedsCount(def id, def count) { DBObject uquery = new BasicDBObject() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy index 3a8ee77b2..c8ab1b6d8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy @@ -61,7 +61,7 @@ class DataStandardBatch extends BaseBatch{ def static final SHORT_DATE = Utils.getLastMonth() def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6] - def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房"] + def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"] public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) static { thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy new file mode 100644 index 000000000..b43b98223 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -0,0 +1,106 @@ +package com.fangjia.script.thirdPriceCrawl + +import com.fangjia.script.util.CommonUtils +import com.mongodb.BasicDBObject +import org.apache.solr.common.util.Hash + +class SynchDistrictPrice { + + def static final SHORT_DATE = Utils.getLastMonth() + def thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); + def districtDetailColl = CrawalDBUtil.getDistrictDetailColl(); + def sourceMap = ["jiwu.com":"吉屋"] + //"58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","jiwu.com":"吉屋" + + static main(args) { + def task = new SynchDistrictPrice() + task.run() + } + + + def run(){ + def citys = ["福州"]//CommonUtils.findAllCity() + + sourceMap.each { k, v -> + for(def city : citys){ + def rets = findCrawlDistrictDetails(city, k, v) + synchData(city, rets, v) + println city +" "+rets.size() + } + } + + } + + def findCrawlDistrictDetails(city, root, source){ + def q = new BasicDBObject() + q.put("city", city) + q.put("root", root) + def keys = new BasicDBObject() + keys.put("city", 1) + keys.put("region", 1) + keys.put("block", 1) + keys.put("name", 1) + keys.put("address", 1) + keys.put("avg_price", 1) + keys.put("root", 1) + keys.put("url", 1) + def rets = [] + def dbcursor = districtDetailColl.find(q, keys) + def names = new HashSet() + for(def d : dbcursor){ + def s = d.city +"#"+d.region+"#"+d.name + if(!names.contains(s)){ + names << s + d.put("district_id", d.url) + d.put("source", source) + d.put("s_date", SHORT_DATE) + d.removeField("root") + d.removeField("url") + d.removeField("_id") + rets << d + } + } + return rets + } + + def synchData(def city, def rets, def source){ + def q = new BasicDBObject() + q.put("source", source) + q.put("s_date", SHORT_DATE) + q.put("city", city) + def count = thirdPartyColl.count(q) + def addIndex = 0 + if(count==0){ + try{ + thirdPartyColl.insert(rets) + }catch(Exception e){ + println e.getMessage() + } + + }else{ + for(def r : rets){ + def flag = findIsExists(r.city, r.region, r.name, SHORT_DATE, source) + if(!flag){ + addIndex++ + thirdPartyColl.save(r) + } + } + } + + println "["+source +"-"+city + "] add:"+ addIndex+" 原抓取:"+count + } + + def findIsExists(def city, def region, def name, def sDate, def source){ + def q = new BasicDBObject() + q.put("source", source) + q.put("s_date", sDate) + q.put("city", city) + q.put("region", region) + q.put("name", name) + def count = thirdPartyColl.count(q) + if(count){ + return true + } + return false + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy index 67e67e520..5f96db29c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy @@ -30,7 +30,7 @@ class ThirdDataToGatherBatch extends BaseBatch{ "房估估":"fanggugu_esf","诸葛找房":"zhugezhaofang_esf","安居客":"anjuke_esf","搜房":"soufang_esf","中原地产":"zhongyuan_esf","赶集":"ganji_esf", "链家":"lianjia_esf","兔博士":"tuboshi_esf","城市房产":"cityhouse_esf","贝壳找房":"beike_esf","Q房":"qfang_esf","Q房(新)":"qfang_new","中原地产(新)":"zhongyuan_new", "乐有家":"leyoujia_esf","乐有家(新)":"leyoujia_new","房探":"fangtan_esf","房多多":"fangdoudou_esf","小资家":"xiaozijia_esf","我爱我家":"5i5j_esf","麦田":"maitian_esf", - "房途":"fangtoo_esf","C21":"c21_esf","乐居":"leju_esf","房产超市":"fccs_esf","58同城":"58tc_esf","365淘房":"365house_esf" + "房途":"fangtoo_esf","C21":"c21_esf","乐居":"leju_esf","房产超市":"fccs_esf","58同城":"58tc_esf","365淘房":"365house_esf","吉屋":"jiwu_esf" ] public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(2) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy index f35379b65..a23842026 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy @@ -309,6 +309,31 @@ public class OpenApiClient { return null } + + public static JSONObject addressStandard(String city, String address) { + if (StringUtils.isBlank(city) || StringUtils.isBlank(address)) { + throw new IllegalArgumentException(); + } + + try { + Map params = new HashMap(); + params.put("city", city); + params.put("address", address); + params.put("token", TOKEN); + String content = new HttpRequest().get("http://open.fangjia.com/address/standard", params, "utf-8") + JSONObject result = JSONObject.fromObject(content) + if (result.getInt("code") == 200) { + return result.getJSONObject("result"); + } else if (result.getInt("code") == 500) { + println("${city} ${address} ${result}") + } + } catch (Exception e) { + println("${city} ${address} ${e.getMessage()}") + e.printStackTrace(); + } + return null + } + public static JSONObject evaluateByGeo(String city, def lat, def lng, String buildDate, String estateType, String geoType) { try { Map params = new HashMap(); @@ -1214,4 +1239,6 @@ public class OpenApiClient { return null; } + + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy index 95d41237b..235253d60 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy @@ -4,6 +4,7 @@ import com.fangjia.core.ProvinceData import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DataUtil import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils @@ -11,45 +12,71 @@ import com.mongodb.BasicDBObject import com.mongodb.DBCollection import net.sf.json.JSONObject import org.apache.commons.lang.StringUtils -DBCollection SEAWEED_TOWNSHIP = MongoUtils.getCollectionByName("fangjia_base", "seaweed_township", "192.168.0.235", 27018) -def count = 0 -List> data = Collections.synchronizedList([]) -data << ["ID","城市","区域","小区名","经度","纬度","街道"] - -CommonUtils.findAllCity().each {def city-> - DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject item-> - if(StringUtils.isBlank(item.township) && item.lng){ - count++ - def lng = item.lng/10000000000d - def lat = item.lat/10000000000d - def location = "${lng},${lat}" - JSONObject resp = GaoDeApiUtils.geocode(location) - def row = [] - row << item._id.toString() - row << item.city - row << item.region - row << item.name - row << lng - row << lat - BasicDBObject dbo = new BasicDBObject() - dbo.id = item._id.toString() - dbo.city = item.city - dbo.region = item.region - dbo.name = item.name - dbo.lng = lng - dbo.lat = lat - if(resp.regeocode && resp.regeocode.addressComponent && resp.regeocode.addressComponent.township){ - row << resp.regeocode.addressComponent.township - dbo.township = resp.regeocode.addressComponent.township - } - println(row) - data << row - SEAWEED_TOWNSHIP.insert(dbo) - } - }) -} -ExcelUtils.write("D:\\街道补充数据.xlsx", data, "补充数据") +//DBCollection SEAWEED_TOWNSHIP = MongoUtils.getCollectionByName("fangjia_base", "seaweed_township", "192.168.0.235", 27018) +//def count = 0 +//List> data = Collections.synchronizedList([]) +//data << ["ID","城市","区域","小区名","经度","纬度","街道"] +// +//CommonUtils.findAllCity().each {def city-> +// DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject item-> +// if(StringUtils.isBlank(item.township) && item.lng){ +// count++ +// def lng = item.lng/10000000000d +// def lat = item.lat/10000000000d +// def location = "${lng},${lat}" +// JSONObject resp = GaoDeApiUtils.geocode(location) +// def row = [] +// row << item._id.toString() +// row << item.city +// row << item.region +// row << item.name +// row << lng +// row << lat +// BasicDBObject dbo = new BasicDBObject() +// dbo.id = item._id.toString() +// dbo.city = item.city +// dbo.region = item.region +// dbo.name = item.name +// dbo.lng = lng +// dbo.lat = lat +// if(resp.regeocode && resp.regeocode.addressComponent && resp.regeocode.addressComponent.township){ +// row << resp.regeocode.addressComponent.township +// dbo.township = resp.regeocode.addressComponent.township +// } +// println(row) +// data << row +// SEAWEED_TOWNSHIP.insert(dbo) +// } +// }) +//} +//ExcelUtils.write("D:\\街道补充数据.xlsx", data, "补充数据") +//ExcelUtils.modify("D:\\街道补充数据.xlsx",{List item -> +// if(item["G"] && item["H"] == null){ +// DbUtils.seaweed.update(new BasicDBObject("_id", item["A"]), new BasicDBObject('$set', new BasicDBObject('township', item["G"]))) +// } +// println(item) +//}) +//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Downloads\\错误经纬度.xlsx",{List item-> +// def location = "${item["A"]},${item["B"]}" +// JSONObject resp = GaoDeApiUtils.geocode(location) +// if(resp.regeocode && resp.regeocode.formatted_address){ +// item['G'] = resp.regeocode.formatted_address +// println(item) +// } +//}) +//def count = 0 +//CommonUtils.findAllCity().each {def city-> +// DbUtils.executeQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject item-> +// if(item.township){ +// count++ +// } +// }, new BasicDBObject("township", 1)) +//} +//println(count) +ElasticUtils.executeSqlQuery("select count(*), p_date from seawater where p_date > 20210501 group by p_date",{ + println(it) +}) \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy index 401d647e4..ff32247d7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy @@ -9,6 +9,11 @@ import com.fangjia.core.util.HttpRequest import com.fangjia.script.util.ExcelUtils import net.sf.json.JSONObject +def strt(String addr){ + +} + + ExcelUtils.modify("D:\\nextcloud\\数据\\地址类服务测试\\建行地址相似度评级测试.xlsx",{List item-> def city = item["A"] def address1 = item["B"] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeMapApiData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeMapApiData.groovy index 4d472474c..c9e44b3e0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeMapApiData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeMapApiData.groovy @@ -223,6 +223,32 @@ class GaoDeMapApiData { doSearchByGaoDe(type, lng, lat, radius, page) } } + + def static doSearchByGaoDe(String type, Double lng, Double lat, int radius, int page, int pageSize) { + Map params = new HashMap() + params.put("key", GAO_DE_AK) + params.put("types", type) + params.put("location", Double.toString(lng)+ ","+ Double.toString(lat)) + params.put("radius",radius) + params.put("offset",pageSize) + params.put("page",page) + String content + try { + content = new HttpRequest().newGet("http://restapi.amap.com/v3/place/around", params, "utf-8") + } catch (Exception e) { + content = new HttpRequest().newGet("http://restapi.amap.com/v3/place/around", params, "utf-8") + } + JSONObject jsonObj = JSONObject.fromObject(content) + + if(jsonObj.get("infocode").equals("10000")){ + JSONArray pois = jsonObj.get("pois") + return pois + } else { + println jsonObj.get("infocode")+"exit........" + doSearchByGaoDe(type, lng, lat, radius, page, pageSize) + } + return null + } private synchronized static fzData(JSONArray pois) { pois.each { poi ->