From e773ca283a84979073d2d0c007dba394af4ecefe Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 19 Aug 2021 19:50:31 +0800 Subject: [PATCH] =?UTF-8?q?=E6=97=A0=E7=BB=8F=E7=BA=AC=E5=BA=A6=E4=B8=8D?= =?UTF-8?q?=E5=90=8C=E6=AD=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../BigdataCommbuildtypeBatch.groovy | 2 +- .../ccbevaluation/BigdataDealBatch.groovy | 2 +- .../ccbevaluation/BigdataForsaleBatch.groovy | 4 +- .../ccbevaluation/BigdataPoiBatch.groovy | 195 +++++++++++ .../ccbevaluation/CcbCommonUtils.groovy | 20 +- .../ccbevaluation/StagingDistrictBatch.groovy | 2 +- .../fangjia/script/ccbevaluation/Test.groovy | 207 +++++------ .../ccbevaluation/ThirdPartyPriceBatch.groovy | 32 +- .../script/ccbevaluation/TradeStatLog.groovy | 3 +- .../com/fangjia/script/fuzhou/Test1.groovy | 2 +- .../com/fangjia/script/gj/RegionPoint.groovy | 12 +- .../fangjia/script/gj/YouDaDataParser.groovy | 24 +- .../gj/data/AddDistrictFieldValue.groovy | 211 +++++++++++- .../script/gj/district/DataProcess.groovy | 4 +- .../gj/district/MapDistrictProcess.groovy | 47 ++- .../district/SeaweedDataMatchProcess.groovy | 3 + .../gj/districtwash/AddDetailDistrict.groovy | 320 ++++++++++++++++++ .../AddThirdDistrictField.groovy | 3 +- .../fangjia/script/gj/location/Point.groovy | 27 ++ .../script/gj/location/PointUtil.groovy | 143 ++++++++ .../gj/test/CrawlDistrictIdSelect.groovy | 12 +- .../com/fangjia/script/gj/test/Test.groovy | 34 +- .../thirdPriceCrawl/SynchDistrictPrice.groovy | 7 +- .../com/fangjia/script/util/DateUtils.groovy | 7 +- .../script/wenyan/RegionTransfer.groovy | 8 +- 25 files changed, 1158 insertions(+), 173 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataPoiBatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/districtwash/AddDetailDistrict.groovy rename projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/{ => districtwash}/AddThirdDistrictField.groovy (98%) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/Point.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/PointUtil.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy index f36fb42b5..c1126334f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy @@ -33,7 +33,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ for(def city : citys){ def cityCodeDatas = CcbCommonUtils.findCityCodeValue() // def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) - def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","build_type", "c_date", "m_date"]) + def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","build_type", "lng","c_date", "m_date"]) def rows = createDatas(cityCodeDatas.get(city), dists) rets.addAll(rows) println city +" "+rows.size() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 2a413d1b3..b56aa4f6b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -390,7 +390,7 @@ class BigdataDealBatch extends BaseBatch{ def getCityData(def city, def statMap){ - def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","build_type","build_structure", "house_property","property_life"]) + def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","lng","build_type","build_structure", "house_property","property_life"]) def tradesList = Collections.synchronizedList(new ArrayList()); def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(dists.size()) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 03c419ca5..72505e112 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -64,14 +64,14 @@ class BigdataForsaleBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = ["陵水"]//CcbCommonUtils.citys; + def citys = ["陵水","文昌","保亭"]//CcbCommonUtils.citys; def index = 0 def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() for(def city : citys){ index++ // def distIds = getDistrictIds(cityCodeDataMap.get(city)) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) - def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]); + def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]); exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20200701, 20210731); } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataPoiBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataPoiBatch.groovy new file mode 100644 index 000000000..70ec7bde0 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataPoiBatch.groovy @@ -0,0 +1,195 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.core.util.GeoUtils +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 建行数据poi同步 + */ +@BatchOwner(Watcher.GaoJuan) +class BigdataPoiBatch extends BaseBatch{ + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80) + + static main(args) { + runBatch(new BigdataPoiBatch(), args); + } + + @Override + public void process(Object args) { + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def citys = CcbCommonUtils.citys; +// def path = "D:\\数据\\建行\\房产估值\\poi\\公交.xlsx" + def subwaydatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\地铁.xlsx", 0,1) + def busDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\公交.xlsx", 0,1) + def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\医院.xlsx", 0,1) + def bankDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\银行.xlsx", 0,1) + def dealDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\丧葬服务.xlsx", 0,1) + def schoolDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\学校.xlsx", 0,1) + + def rets = [] + def rows1 = createDatas(citys, cityCodeDataMap, subwaydatas, "地铁") + rets.addAll(rows1) + + def rows2 = createDatas(citys, cityCodeDataMap, busDatas, "公交") + rets.addAll(rows2) +// + def rows3 = createDatas(citys, cityCodeDataMap, hosptialDatas, "医院") + rets.addAll(rows3) +// + def rows4 = createDatas(citys, cityCodeDataMap, bankDatas, "银行") + rets.addAll(rows4) + + def rows5 = createDatas(citys, cityCodeDataMap, dealDatas, "丧葬服务") + rets.addAll(rows5) + + def rows6 = createDatas(citys, cityCodeDataMap, schoolDatas, "学校") + rets.addAll(rows6) + + def headers = ["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","来源公司"] + + def fileName = "/home/goojia/ccb/bigdata_poi.txt" + if(CommonDatautils.isWindow()){ + fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_poi.txt" + } + FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false); + println "txt 生成" + + +// def headers1 = ["城市ID","城市","行政区ID","区域名称","经度","纬度","创建时间","来源公司"] +// def regionCenterDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\行政区中心点经纬度.xlsx", 0,1) +// def regionRets = createRegionCenters(regionCenterDatas) +// FileUtils.batchCreateFile(regionRets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\region_center.txt", headers1, "utf-8", false); +// println "行政区域中心txt 生成" + + } + + def createRegionCenters(def datas){ + def rets = [] + for(def d : datas){ + def ret = [] + ret << d['A'].toInteger().toString() + ret << d['B'] + ret << d['C'].toInteger().toString() + ret << d['D'] + def loactionStr = d['E'] + if(loactionStr){ + def arrs = loactionStr.split(",") + def loactionMap = GeoUtils.gcjToGps84(arrs[1].toDouble(), arrs[0].toDouble()) + ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lng,6) + ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6) + + }else{ + ret << "" + ret << "" + } + ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + ret << "FJW" + rets << ret + } + return rets; + + } + + def createDatas(citys, cityCodeDataMap, datas, type){ + def rets = Collections.synchronizedList(new ArrayList()); + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(datas.size()) + def cityDistMap = CcbCommonUtils.findAllCityDistrictCodeValue() + for(def i=0;i +def cityCodeDatas = CcbCommonUtils.findCityCodeValue() +//小区量 +//ExcelUtils.multiThreadingModify(fileName, { d -> // def city = d['B'] -// if(city){ +// if (city) { // def cityid = cityCodeDatas.get(city) // def total = findCount(cityid, sqlInstance) -//// d['I'] = total -//// def a = findEstateType(cityid, sqlInstance) -//// -//// d['AA'] = total -//// d['AB'] = DataHandleUtils.getDoubleRound(2, a*100/total) -//// -//// def b = findFiledCount(cityid, 'buildyear', sqlInstance) -//// d['AC'] = DataHandleUtils.getDoubleRound(2, b*100/total) -//// -//// def c = findFiledCount(cityid, 'address', sqlInstance) -//// d['AD'] = DataHandleUtils.getDoubleRound(2, c*100/total) -//// -//// def d1 = findFiledCount(cityid, 'ZTS', sqlInstance) -//// d['AE'] = DataHandleUtils.getDoubleRound(2, d1*100/total) -//// -//// def e = findFiledCount(cityid, 'landarea', sqlInstance) -//// d['AF'] = DataHandleUtils.getDoubleRound(2, e*100/total) -//// -//// def f = findFiledCount(cityid, 'buildarea', sqlInstance) -//// d['AG'] = DataHandleUtils.getDoubleRound(2, f*100/total) -//// -//// def g = findFiledCount(cityid, 'plotratio', sqlInstance) -//// d['AH'] = DataHandleUtils.getDoubleRound(2, g*100/total) -//// -//// def h = findFiledCount(cityid, 'greeningrate', sqlInstance) -//// d['AI'] = DataHandleUtils.getDoubleRound(2, h*100/total) -//// -//// def i = findFiledCount(cityid, 'latitude', sqlInstance) -//// d['AJ'] = DataHandleUtils.getDoubleRound(2, i*100/total) -//// -//// def j = findFiledCount(cityid, 'parkingcnt', sqlInstance) -//// d['AK'] = DataHandleUtils.getDoubleRound(2, j*100/total) -// -// println d +// d['K'] = total // } //},0) + +//小区字段覆盖率 +ExcelUtils.multiThreadingModify(fileName,{d-> + def city = d['A'] + if(city){ + def cityid = cityCodeDatas.get(city) + def total = findCount(cityid, sqlInstance) + d['AA'] = total + + def a = findEstateType(cityid, sqlInstance) + d['AB'] = DataHandleUtils.getDoubleRound(2, a*100/total) + + def b = findFiledCount(cityid, 'buildyear', sqlInstance) + d['AC'] = DataHandleUtils.getDoubleRound(2, b*100/total) + + def c = findFiledCount(cityid, 'address', sqlInstance) + d['AD'] = DataHandleUtils.getDoubleRound(2, c*100/total) + + def d1 = findFiledCount(cityid, 'ZTS', sqlInstance) + d['AE'] = DataHandleUtils.getDoubleRound(2, d1*100/total) + + def e = findFiledCount(cityid, 'landarea', sqlInstance) + d['AF'] = DataHandleUtils.getDoubleRound(2, e*100/total) + + def f = findFiledCount(cityid, 'buildarea', sqlInstance) + d['AG'] = DataHandleUtils.getDoubleRound(2, f*100/total) + + def g = findFiledCount(cityid, 'plotratio', sqlInstance) + d['AH'] = DataHandleUtils.getDoubleRound(2, g*100/total) + + def h = findFiledCount(cityid, 'greeningrate', sqlInstance) + d['AI'] = DataHandleUtils.getDoubleRound(2, h*100/total) + + def i = findFiledCount(cityid, 'latitude', sqlInstance) + d['AJ'] = DataHandleUtils.getDoubleRound(2, i*100/total) + + def j = findFiledCount(cityid, 'parkingcnt', sqlInstance) + d['AK'] = DataHandleUtils.getDoubleRound(2, j*100/total) + println d + } +},1) + //def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0807-1.xlsx" //def dealMap = findDealCount(sqlInstance) //ExcelUtils.multiThreadingModify(fileName,{d-> @@ -206,64 +213,64 @@ def findFiledCount(cityid, field, sqlInstance){ } -def fileName = "d:\\数据\\建行\\房产估值\\经纬度错误数据修正.xlsx" -def updateIndex = 0 -def delIndex = 0 -def cityMap = [:] -ExcelUtils.multiThreadingModify(fileName,{d-> - if(!d['R']){ - - def obj = CommonDatautils.findDistrictById(d['D']) - if(obj){ - def ranges = cityMap.get(obj.city) - if(!ranges){ - def dbcursor = POLYLINECOLL.find(new BasicDBObject("city",obj.city)) - ranges = [] - for(def r : dbcursor){ - ranges << r - } - cityMap.put(obj.city, ranges) - } - - if(ranges){ - def lng = obj.lng/10000000000.0 - def lat = obj.lat/10000000000.0 - - def region = null - for(def r : ranges){ - try{ - def flag = isInArea(lng, lat, r.polyline) - if(flag){ - def regionStr = r.region - region = RegionData.formatRegion(obj.city, regionStr) - break - } - }catch(Exception e){ - println e.getMessage() - } - - } - if(region && region!=obj.region){ - updateIndex++ - DbUtils.seaweed.update(new BasicDBObject("_id",obj.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region))) - println updateIndex +" [update]old:${obj.region} new:"+region+" "+obj.name - }else if(region && region==obj.region){ - println "[same] "+obj.city +" "+obj.name - }else{ - DbUtils.seaweed.update(new BasicDBObject("_id",new ObjectId(d['D'])),new BasicDBObject('$set', new BasicDBObject("status", 1))) - delIndex++ - println delIndex +" [delete] "+obj.city +" "+obj.name - } - }else{ - println obj.city +" 没有此城市边框信息" - } - - }else{ - println d['D']+" 没找到该小区" - } - - } -},0); +//def fileName = "d:\\数据\\建行\\房产估值\\经纬度错误数据修正.xlsx" +//def updateIndex = 0 +//def delIndex = 0 +//def cityMap = [:] +//ExcelUtils.multiThreadingModify(fileName,{d-> +// if(!d['R']){ +// +// def obj = CommonDatautils.findDistrictById(d['D']) +// if(obj){ +// def ranges = cityMap.get(obj.city) +// if(!ranges){ +// def dbcursor = POLYLINECOLL.find(new BasicDBObject("city",obj.city)) +// ranges = [] +// for(def r : dbcursor){ +// ranges << r +// } +// cityMap.put(obj.city, ranges) +// } +// +// if(ranges){ +// def lng = obj.lng/10000000000.0 +// def lat = obj.lat/10000000000.0 +// +// def region = null +// for(def r : ranges){ +// try{ +// def flag = isInArea(lng, lat, r.polyline) +// if(flag){ +// def regionStr = r.region +// region = RegionData.formatRegion(obj.city, regionStr) +// break +// } +// }catch(Exception e){ +// println e.getMessage() +// } +// +// } +// if(region && region!=obj.region){ +// updateIndex++ +// DbUtils.seaweed.update(new BasicDBObject("_id",obj.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region))) +// println updateIndex +" [update]old:${obj.region} new:"+region+" "+obj.name +// }else if(region && region==obj.region){ +// println "[same] "+obj.city +" "+obj.name +// }else{ +// DbUtils.seaweed.update(new BasicDBObject("_id",new ObjectId(d['D'])),new BasicDBObject('$set', new BasicDBObject("status", 1))) +// delIndex++ +// println delIndex +" [delete] "+obj.city +" "+obj.name +// } +// }else{ +// println obj.city +" 没有此城市边框信息" +// } +// +// }else{ +// println d['D']+" 没找到该小区" +// } +// +// } +//},0); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index 179daf27e..f14ace3a9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -38,7 +38,7 @@ class ThirdPartyPriceBatch extends BaseBatch { def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) - def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2"]) + def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"]) def thirdPriceMap = findThirdPartyPrice(city) def rets = createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap) rows.addAll(rets) @@ -180,58 +180,58 @@ class ThirdPartyPriceBatch extends BaseBatch { dataMap.put("id", db._id.toString()) map.put(key, dataMap) } - if(!dataMap.containsKey("zhugezhaofang_esf_price") && db.zhugezhaofang_esf_price){ + if(!dataMap.containsKey("zhugezhaofang_esf_price") && db.zhugezhaofang_esf_price && db.zhugezhaofang_esf_price>3000){ dataMap.put("zhugezhaofang_esf_price", db.zhugezhaofang_esf_price) } - if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price){ + if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){ dataMap.put("anjuke_esf_price", db.anjuke_esf_price) } - if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price")){ + if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){ dataMap.put("anjuke_esf_price", db.get("58tc_esf_price")) } - if(!dataMap.containsKey("soufang_esf_price") && db.soufang_esf_price){ + if(!dataMap.containsKey("soufang_esf_price") && db.soufang_esf_price && db.soufang_esf_price>3000){ dataMap.put("soufang_esf_price", db.soufang_esf_price) } - if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price){ + if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){ dataMap.put("lianjia_esf_price", db.lianjia_esf_price) } - if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price){ + if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){ dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price) } - if(!dataMap.containsKey("maitian_esf_price") && db.maitian_esf_price){ + if(!dataMap.containsKey("maitian_esf_price") && db.maitian_esf_price && db.maitian_esf_price>3000){ dataMap.put("maitian_esf_price", db.maitian_esf_price) } - if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price){ + if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){ dataMap.put("beike_esf_price", db.beike_esf_price) } - if(!dataMap.get("365house_esf_price") && db.get("365house_esf_price")){ + if(!dataMap.get("365house_esf_price") && db.get("365house_esf_price") && db.get("365house_esf_price")>3000){ dataMap.put("365house_esf_price", db.get("365house_esf_price")) } - if(!dataMap.containsKey("zhongyuan_esf_price") && db.zhongyuan_esf_price){ + if(!dataMap.containsKey("zhongyuan_esf_price") && db.zhongyuan_esf_price && db.zhongyuan_esf_price>3000){ dataMap.put("zhongyuan_esf_price", db.zhongyuan_esf_price) } - if(!dataMap.containsKey("tuboshi_esf_price") && db.tuboshi_esf_price){ + if(!dataMap.containsKey("tuboshi_esf_price") && db.tuboshi_esf_price && db.tuboshi_esf_price>3000){ dataMap.put("tuboshi_esf_price", db.tuboshi_esf_price) } - if(!dataMap.containsKey("qfang_esf_price") && db.qfang_esf_price){ + if(!dataMap.containsKey("qfang_esf_price") && db.qfang_esf_price && db.qfang_esf_price>3000){ dataMap.put("qfang_esf_price", db.qfang_esf_price) } - if(!dataMap.containsKey("fangtan_esf_price") && db.fangtan_esf_price){ + if(!dataMap.containsKey("fangtan_esf_price") && db.fangtan_esf_price && db.fangtan_esf_price>3000){ dataMap.put("fangtan_esf_price", db.fangtan_esf_price) } - if(!dataMap.containsKey("fangduoduo_esf_price") && db.fangduoduo_esf_price){ + if(!dataMap.containsKey("fangduoduo_esf_price") && db.fangduoduo_esf_price && db.fangduoduo_esf_price>3000){ dataMap.put("fangduoduo_esf_price", db.fangduoduo_esf_price) } - if(!dataMap.get("5i5j_esf_price") && db.get("5i5j_esf_price")){ + if(!dataMap.get("5i5j_esf_price") && db.get("5i5j_esf_price") && db.get("5i5j_esf_price")>3000){ dataMap.put("5i5j_esf_price", db.get("5i5j_esf_price")) } // if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index b5f10ed40..e7bd95d9d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -25,6 +25,7 @@ class TradeStatLog { def task = new TradeStatLog() //task.run() task.thirdPartyStat() +// println Calendar.getInstance().get(Calendar.YEAR) } @@ -88,7 +89,7 @@ class TradeStatLog { // def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() rets << ["城市","小区总量","有第三方的总量","1家占比","2家占比","3家占比","4家占比","5家占比","6家占比","7家占比","8家占比","9家占比","10家占比","11家占比","12家占比","13家占比","14家占比"] for(def city : citys){ - def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2", "name_cat"]); + def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2", "name_cat","lng"]); def m = getThirdPartyPricePrecent(city) rets << [city, districts.size(), m.total, m.count1, m.count2, m.count3, m.count4, m.count5, m.count6, m.count7, m.count8, m.count9, m.count10, m.count11, m.count12, m.count13, m.count14] println rets.get(rets.size()-1) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index 0a1582501..c61861c99 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -14,7 +14,7 @@ import com.mongodb.DBCollection //福州第三方均价统计,发送钉钉消息 //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); -def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202106.csv") +def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202107.csv") def count0 = 0 def count1 = 0 def count2 = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/RegionPoint.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/RegionPoint.groovy index 4d603f258..efa1ead14 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/RegionPoint.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/RegionPoint.groovy @@ -1,6 +1,7 @@ package com.fangjia.script.gj import com.fangjia.core.Constants +import com.fangjia.core.RegionData import com.fangjia.core.util.HttpRequest import com.fangjia.core.util.Mongo3Utils import com.fangjia.script.commData.CommonDatautils @@ -29,7 +30,8 @@ class RegionPointCrawl { def task = new RegionPointCrawl(); //河北省、山西省、辽宁省、吉林省、黑龙江省、江苏省、浙江省、 // def provinces = ["安徽省","福建省","江西省","山东省","河南省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省","内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"] - def provinces = ["浙江省"] + //def provinces = ["河北省","山西省","辽宁省","吉林省","黑龙江省","江苏省","浙江省","安徽省","福建省","江西省","山东省","河南省","湖北省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省"] + def provinces = ["内蒙古自治区","广西壮族自治区","新疆维吾尔自治区"] for(def p : provinces){ task.crawlCityAndDistrictCode(p); task.run(p); @@ -55,7 +57,13 @@ class RegionPointCrawl { } println index +" "+cityMap.name+" "+m.name+" "+ret.size() if(ret.size()>0){ - saveData(cityMap.name, m.name, m.ad_code , ret); + def standardCity = RegionData.formatCity(cityMap.name) + if(standardCity){ + saveData(standardCity, m.name, m.ad_code , ret); + }else{ + println cityMap.name +" 没有标准化" + } + } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index d8ccfa02e..c2c545471 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -6,6 +6,7 @@ import com.fangjia.core.util.HttpRequest import com.fangjia.httpclient.UserAgents import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.district.SeaweedDataMatchProcess import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils @@ -47,14 +48,13 @@ class YouDaDataParser { def final AtomicInteger count_ = new AtomicInteger(0) - static def statDate = "2021-06-01" + static def statDate = "2021-07-01" static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo - task.esfParser();//友达二手房存入mongo +// task.esfParser();//友达二手房存入mongo // task.newhouseToSaveEs();//新房保存到es task.esfhouseToSaveEs();//二手房保存到es - // task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间 } @@ -285,7 +285,7 @@ class YouDaDataParser { def esfParser(){ - def files = ["D:\\数据\\友达\\2021年7月\\二手房\\查询1.DBF"] + def files = ["D:\\数据\\友达\\2021年8月\\二手房\\查询1.DBF"] def datas = [] for(def fileName : files){ def rows = readDBF(fileName); @@ -308,10 +308,7 @@ class YouDaDataParser { } ); } - latch.await() - - } def saveEsfHouseToDB(def data){ @@ -367,15 +364,16 @@ class YouDaDataParser { row.ydId = data[2].toInteger() row.fjCity = city - def map = matchDistrict(city, row.region, row.districtName) - if(!map){ - map = matchDistrict(city, row.region, row.address) - } +// def map = matchDistrict(city, row.region, row.districtName) +// if(!map){ +// map = matchDistrict(city, row.region, row.address) +// } + def map = SeaweedDataMatchProcess.doMatch(city, row.region, row.districtName, row.address); if(map){ row.fjRegion = map.region row.fjName = map.name - row.lat = map.lat - row.lng = map.lng + row.lat = map.lat.toDouble() + row.lng = map.lng.toDouble() row.location = map.lat.toString()+","+map.lng.toString() def obj = CommonDatautils.findDistrictById(map.id.toString()); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/AddDistrictFieldValue.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/AddDistrictFieldValue.groovy index d91c34642..ed07e1dee 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/AddDistrictFieldValue.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/AddDistrictFieldValue.groovy @@ -7,16 +7,22 @@ import com.fangjia.script.gj.FiledStrand import com.fangjia.script.gj.district.SeaweedDataMatchProcess import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil import com.fangjia.script.util.DataUtil +import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils import com.mongodb.BasicDBObject import com.mongodb.DBObject import com.rabbitmq.client.AMQP import com.rabbitmq.client.impl.AMQImpl import org.bson.types.ObjectId +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicLong class AddDistrictFieldValue { @@ -30,18 +36,26 @@ class AddDistrictFieldValue { } def run(){ - def citys = CcbCommonUtils.citys; + def citys = ["沈阳","哈尔滨","吉林","遵义","淄博","西宁","青岛","大连","宁波"]//CcbCommonUtils.citys; + //第三方均价数据匹配 // for(def city : citys){ // def rets = findAllDataByCity(city) // match(rets) // } - for(def city : citys){ - def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat"]) - def rMap = getMatchDistrict(city) - createResultValue(dists, rMap) - } + //第三方均价数据填补 +// for(def city : citys){ +// def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat"]) +// def rMap = getMatchDistrict(city) +// createResultValue(dists, rMap) +// } + // + for(def city : citys){ + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat","open_date","build_date","live_date", "build_type"]) +// addFinishDate(dists) + addFieldValueByHouse(dists) + } } def match(def rets){ @@ -118,6 +132,35 @@ class AddDistrictFieldValue { return rMap; } + def addFinishDate(dists){ + def updateIndex = 0 + def total = dists.size() + for(def d : dists){ + if(!d.finish_building_date){ + def udata = new BasicDBObject() + def newValue = FiledStrand.processFinishBuildingDate(d.live_date) + if(!newValue && d.open_date){ + def date = d.open_date + newValue = date.getYear()+1900 + } + if(!newValue && d.build_date){ + newValue = FiledStrand.processFinishBuildingDate(d.build_date) + } + if(newValue){ + udata.put("finish_building_date", newValue) + } + if(udata.size()>0){ + updateIndex++ + def q = new BasicDBObject() + q.put("_id", d._id) + println updateIndex +"/${total} "+d.city +" "+d.name+" "+udata + DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) + + } + } + } + } + def createResultValue(dists, rMap){ def updateIndex = 0 def total = dists.size() @@ -260,9 +303,163 @@ class AddDistrictFieldValue { } } } - return lastValue; } + def addFieldValueByHouse(def dists){ + def updateIndex = new AtomicLong() + + def totalCount = new AtomicLong() + def total = dists.size() + final CountDownLatch latch = new CountDownLatch(total) + + for(def i=0;i6 && max<=11){ + udata.put("build_type", "小高层") + }else if(max>11 && max<30){ + udata.put("build_type", "高层") + }else if(max>30){ + udata.put("build_type", "超高层") + } + } + } + } + } + if(udata.size()>0){ + updateIndex.addAndGet(1) + def q = new BasicDBObject() + q.put("_id", d._id) + println updateIndex.get() +"/${total} "+d.city +" "+d.name+" "+udata + DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) + } + }else{ + println totalCount.get() +" "+d.city +" "+d.name + } + + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + } + }); + } + latch.await() + latch.await(5, TimeUnit.MINUTES); + } + + + def findHouses(def city, def region, def cat, def name, def alias){ + +// QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false); + QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)) +// must(QueryBuilders.termQuery("cat", cat)).must(area_range) + + if(!alias){ + alias = [] + } + if(!alias.contains(name)){ + alias.add(name) + } + + + if(name.contains("弄小区")){ + def newName = name.replace("弄小区","弄") + if(!alias.contains(newName)){ + alias << newName + } + + } + + if(alias && alias.size()>0){ + BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); + for(def s : alias){ + inBuilder.should(QueryBuilders.matchQuery("district_name", s)); + } + queryBuilder.must(inBuilder) + } + + def restMap = [:] + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-*", queryBuilder); + if(houses){ + for (int k=0;k b.value <=> a.value} + restMap.put("build_date", dateMap) + } + } + return restMap; + } + + + + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DataProcess.groovy index 51632fe04..2f9eb809e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DataProcess.groovy @@ -24,7 +24,7 @@ class DataProcess { static main(args) { //建行信用卡城市(临汾--晋城).xlsx - def filePath = "D:\\数据\\小区\\地图\\张家口.xlsx" + def filePath = "D:\\数据\\小区\\地图\\兰州.xlsx" def task = new DataProcess(); // task.batchSaveDataToDB(filePath) task.mapBatchSaveDataToDB(filePath) @@ -121,7 +121,7 @@ class DataProcess { if(datas){ def keys = [] for(def data : datas){//20,21,25, - if(data['I'] && [20,21,25,26,27].contains(data['I'].toString().toDouble().toInteger())){ + if(data['I'] && [20,21,26,27].contains(data['I'].toString().toDouble().toInteger())){ def key = data['A']+"##"+data['B']+"##"+data['H'] if(!keys.contains(key) && data['H'] && data['H'].size()>2){ keys << key diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/MapDistrictProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/MapDistrictProcess.groovy index ca4e29905..49216581d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/MapDistrictProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/MapDistrictProcess.groovy @@ -5,15 +5,22 @@ import com.fangjia.core.util.BaiDuApiUtils import com.fangjia.core.util.DataUtil import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.EsSearchUtils +import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil import com.fangjia.script.util.CosineSimilarAlgorithm import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.GeoHelper import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.fangjia.script.yinjihaun.lng.UpdateDistrictLocation +import com.mongodb.BasicDBObject import java.util.regex.Pattern +/*** + * 地图小区处理 + */ class MapDistrictProcess { + def districtDetailColl = CrawalDBUtil.getDistrictDetailColl(); + static main(args) { def task = new MapDistrictProcess(); task.run() @@ -28,17 +35,18 @@ class MapDistrictProcess { def run(){ // def files = ["重庆.xlsx","东莞.xlsx","温州.xlsx","绍兴.xlsx","宁波.xlsx","南昌.xlsx","济南.xlsx","金华.xlsx","肇庆.xlsx","湛江.xlsx","岳阳.xlsx","盐城.xlsx","唐山.xlsx","三亚.xlsx","廊坊.xlsx","呼和浩特.xlsx","西宁.xlsx","日照.xlsx","常德.xlsx"] - def files = ["青岛.xlsx","沈阳.xlsx","哈尔滨.xlsx","张家口.xlsx","大连.xlsx"] + //"乌鲁木齐.xlsx","烟台.xlsx","保定.xlsx","贵阳.xlsx","湖州.xlsx","惠州.xlsx","吉林.xlsx","遵义.xlsx","宜昌.xlsx","芜湖.xlsx","兰州.xlsx" + def files = ["兰州.xlsx"] for(def fileName : files){ def filePath = "D:\\数据\\小区\\地图\\"+fileName; typeFilter(filePath) nameStandard(filePath); dataFilter(filePath) houseCalculate(filePath); -////// //统计是否存在多个区域 + //统计是否存在多个区域 matchDistDatas(filePath) moreRegionCalculate(filePath) - +// addFlag(filePath) mapMatch(filePath); baiduTypeFilter(filePath); @@ -50,7 +58,8 @@ class MapDistrictProcess { addFeature(filePath) houseMatch(filePath); -// nameScore(filePath); + nameScore(filePath); + thirdPartyData(filePath) } } @@ -212,7 +221,7 @@ class MapDistrictProcess { def addFlag(def filePath){ def index = 0 - ExcelUtils.modify(filePath, { row -> + ExcelUtils.multiThreadingModify(filePath, { row -> index++ if(!row['I']) { //根据别名查找 @@ -438,6 +447,34 @@ class MapDistrictProcess { },0) } + def thirdPartyData(filePath){ + def districtDetailColl = CrawalDBUtil.getDistrictDetailColl(); + ExcelUtils.multiThreadingModify(filePath, { row -> + if(row['I'] && [20,21,25,26,27].contains(row['I'].toString().toDouble().toInteger())){ + def rets = findIsExistsOtherWebsite(row['A'], row['B'], row['H']) + if(rets){ + row['W'] = rets.join(",") + } + } + + },0) + + } + + def findIsExistsOtherWebsite(def city, def region, def name){ + def q = new BasicDBObject() + q.put("city",city) + q.put("region",region) + q.put("name", name) + def dbcursor = districtDetailColl.find(q) + def rets = [] + for(def r : dbcursor){ + if(!rets.contains(r.root)){ + rets << r.root + } + } + return rets + } def getRegionByLocation(lng, lat){ def jsonObject = GaoDeApiUtils.geocode(lng.toString()+","+lat.toString()) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy index 96b55d4c7..b6d403059 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy @@ -25,6 +25,9 @@ class SeaweedDataMatchProcess { rMap.city = obj.city rMap.region = obj.region rMap.name = obj.name + rMap.lat = obj.lat/10000000000.0 + rMap.lng = obj.lng/10000000000.0 + rMap.id = obj._id.toString() } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/districtwash/AddDetailDistrict.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/districtwash/AddDetailDistrict.groovy new file mode 100644 index 000000000..a6286194a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/districtwash/AddDetailDistrict.groovy @@ -0,0 +1,320 @@ +package com.fangjia.script.gj.districtwash + +import com.fangjia.core.RegionData +import com.fangjia.core.util.GeocodeUtils +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.FiledStrand +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.gj.district.NameProcess +import com.fangjia.script.gj.district.SeaweedDataMatchProcess +import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil +import com.fangjia.script.util.DataUtil +import com.fangjia.script.util.DbUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject + +class AddDetailDistrict { + + def detailColl = CrawalDBUtil.getDistrictDetailColl(); + + static main(args) { + def task = new AddDetailDistrict(); +// task.run(); +// task.delRepeatDist + task.addEstateType() + } + + def addEstateType(){ + def citys = ["沈阳","哈尔滨","吉林","遵义","淄博","西宁","青岛","大连","宁波"] + for(def city : citys){ + def keys = [] + def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","estate_type2"]) + for(def d : districts) { + if(!d.estate_type2){ + def estateType2 = "普通住宅" + if(d.name.contains("别墅")){ + estateType2 = "别墅" + }else if(d.name.contains("商住")){ + estateType2 = "商住" + } + DBObject update = new BasicDBObject(); + update.put('$set', new BasicDBObject("estate_type2", estateType2).append("m_date", new Date())); + DbUtils.seaweed.update(new BasicDBObject("_id", d.get("_id")), update) + } + } + println city + } + } + + def delRepeatDist(){ + def citys = CcbCommonUtils.citys + for(def city : citys){ + def keys = [] + def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name"]) + def index = 0 + for(def d : districts){ + index++ + def key = d.city +"##"+d.region+"##"+d.name + if(!keys.contains(key)){ + keys << key + }else{ + println index +" "+d.region+" "+d.name + DBObject update = new BasicDBObject(); + update.put('$set', new BasicDBObject("status", 1).append("m_date", new Date())); + DbUtils.seaweed.update(new BasicDBObject("_id", d.get("_id")), update) + } + } + } + + } + + def run() { + def citys = ["沈阳", "哈尔滨", "吉林", "遵义", "淄博", "西宁", "青岛", "大连", "宁波"]//CcbCommonUtils.citys; + //第三方均价数据匹配 + for (def city : citys) { + def rets = findAllDataByCity(city) + def rMap = getMatchDistrict(city) + toDB(rets, rMap) + } + } + + def toDB(def rets, def rMap){ + def saveDistKeys = [] + def total = rets.size() + def index = 0 + def addIndex = 0 + for(def d : rets){ + index++ + def newRegion = RegionData.formatRegion(d.city, d.region) + def name = NameProcess.nameStandard(d.city, newRegion, d.name) + if(!name){ + continue + } + def flag = NameProcess.filterName(name) + def key = null + if(newRegion){ + key = d.city+"#"+newRegion+"#"+name + } + if(flag && !saveDistKeys.contains(key)){ + def gdData = FangjiaClientUtils.getMapData(d.city, newRegion, name, d.address,"", "高德") + def lng2 = null + def lat2 = null + def lng = null + def lat = null + if(gdData){ + + if(!newRegion){ + newRegion = RegionData.formatRegion(d.city, gdData.district) + if(newRegion){ + key = d.city+"#"+newRegion+"#"+name + } + } + } + + if(key && gdData && gdData.lng){ + def matchMap = SeaweedDataMatchProcess.doMatch(d.city, newRegion, name, d.address); + if(!matchMap){ + addIndex++ + if(gdData.lng){ + def gmap = GeocodeUtils.getGaodeToBaidu(gdData.lng.toDouble(), gdData.lat.toDouble()) + lng2 = (gmap.get("lng").toDouble()* 10000000000).longValue() + lat2 = (gmap.get("lat").toDouble()* 10000000000).longValue() + lng = (gdData.lng.toDouble()* 10000000000).longValue() + lat = (gdData.lat.toDouble()* 10000000000).longValue() + } + def newAddress = DataUtil.formateAddress(d.city, d.address) + def udata = createFieldValue(d, rMap) + udata.put("lng", lng) + udata.put("lat", lat) + println index+"/${addIndex}/${total} "+d.city+" "+newRegion+" "+name+" "+d.root+" "+udata + udata.put("city", d.city) + udata.put("region", newRegion) + udata.put("name", name) + udata.put("address", newAddress) + + udata.put("lng2", lng2) + udata.put("lat2", lat2) + udata.put("status", 0) + udata.put("cat", "district") + udata.put("c_date", new Date()) + udata.put("m_date", new Date()) + DbUtils.seaweed.save(udata) + saveDistKeys << key + } + } +// println d.city+" "+ d.region+" "+d.name + } + } + } + + def createFieldValue(d, rMap){ + def key = d.city +"#"+d.region+"#"+d.name + def sourceMap = rMap.get(key) + def udata = new BasicDBObject() + if(sourceMap){ + if(!d.address){ + def lastValue = getValuesByField(sourceMap,"address", d.estate_type2) + if(lastValue){ + udata.put("address", lastValue) + } + } + if(!d.all_area){ + def lastValue = getValuesByField(sourceMap,"all_area", d.estate_type2) + if(lastValue){ + udata.put("all_area", lastValue) + } + } + + if(!d.build_area){ + def lastValue = getValuesByField(sourceMap,"build_area", d.estate_type2) + if(lastValue){ + udata.put("build_area", lastValue) + } + } + + if(!d.virescence_percent){ + def lastValue = getValuesByField(sourceMap,"greening_rate", d.estate_type2) + if(lastValue){ + udata.put("virescence_percent", lastValue) + } + } + + if(!d.cubage_percent){ + def lastValue = getValuesByField(sourceMap,"volumetric_rate", d.estate_type2) + if(lastValue){ + udata.put("cubage_percent", lastValue) + } + } + + if(!d.finish_building_date){ + def lastValue = getValuesByField(sourceMap,"build_date", d.estate_type2) + if(lastValue){ + udata.put("finish_building_date", lastValue) + } + } + + if(!d.household_count){ + def lastValue = getValuesByField(sourceMap,"house_count", d.estate_type2) + if(lastValue){ + udata.put("household_count", lastValue) + } + } + + if(!d.car_site_count){ + def lastValue = getValuesByField(sourceMap,"car_site", d.estate_type2) + if(lastValue){ + udata.put("car_site_count", lastValue) + } + } + } + return udata + } + + def getValuesByField(sourceMap, field, estateType){ + def map = [:] + sourceMap.each{k,v-> + def oldValue = v.get(field); + if(oldValue){ + def newValue = null; + if(["address"].contains(field)){ + newValue = DataUtil.formateAddress(v.get("city"), oldValue) + } + if(["all_area","build_area"].contains(field)){ + newValue = FiledStrand.processArea(oldValue) + } + if(["build_date"].contains(field)){ + newValue = FiledStrand.processFinishBuildingDate(oldValue) + } + if(["house_count"].contains(field)){ + newValue = FiledStrand.processHouseholdCount(oldValue, estateType) + } + if(["greening_rate"].contains(field)){ + newValue = FiledStrand.processVirescencePercent(oldValue) + } + if(["volumetric_rate"].contains(field)){ + newValue = FiledStrand.processCubagePercent(oldValue, estateType) + } + if(["car_site_desc","car_site"].contains(field)){ + newValue = FiledStrand.processCarSite(oldValue) + } + if(newValue){ + def count = map.get(newValue) + if(!count){ + map.put(newValue, 1) + }else{ + map.put(newValue, count+1) + } + + v.put(field, newValue) + } + } + } + + def lastValue = null + if(map && map.size()>0){ + map = map.sort {a, b -> b.value <=> a.value} + def values = map.values() + if(values.size()==1){ + lastValue = map.keySet()[0] + }else{ + def count1 = map.values()[0] + def count2 = map.values()[1] + if(count1 > count2){ + lastValue = map.keySet()[0] + }else{ + def rets = [map.keySet()[0],map.keySet()[1]] + def roots = ["lianjia.com", "ke.com","fang.com","58.com","anjuke.com","5i5j.com","jiwu.com","centanet.com","house365.com","zhuge.com"] + for(def root : roots){ + def data = sourceMap.get(root) + if(data){ + def value = data.get(field) + if(rets.contains(value)){ + lastValue = value + break + } + } + } + } + } + } + return lastValue; + } + + + def getMatchDistrict(def city){ + def q = new BasicDBObject() + q.put("city", city) +// q.put("match_name", new BasicDBObject('$exists', true)) + def dbcursor = detailColl.find(q) + def rMap = [:] + for(def d : dbcursor){ + def key = d.city +"#"+d.region+"#"+d.name + def sourceMap = rMap.get(key) + if(!sourceMap){ + sourceMap = [:] + rMap.put(key, sourceMap) + } + def value = sourceMap.get(d.root) + if(!value){ + sourceMap.put(d.root, d) + } + } + return rMap; + } + + + def findAllDataByCity(def city){ + def q = new BasicDBObject("city", city); + q.put("match_name", new BasicDBObject('$exists', false)) + q.put("avg_price", new BasicDBObject('$gt', 0)) + + def dbcursor = detailColl.find(q).batchSize(2000) + def rets = [] + for(def d : dbcursor){ + rets << d + } + return rets; + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddThirdDistrictField.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/districtwash/AddThirdDistrictField.groovy similarity index 98% rename from projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddThirdDistrictField.groovy rename to projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/districtwash/AddThirdDistrictField.groovy index 4e81aa072..0d84f4f75 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddThirdDistrictField.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/districtwash/AddThirdDistrictField.groovy @@ -1,8 +1,9 @@ -package com.fangjia.script.gj +package com.fangjia.script.gj.districtwash import com.fangjia.core.RegionData import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.datastore.FieldValidatorUtils +import com.fangjia.script.gj.OpenApiUtils import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil import com.fangjia.script.thirdPriceCrawl.Utils import com.fangjia.script.util.DbUtils diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/Point.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/Point.groovy new file mode 100644 index 000000000..2c3b8d05e --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/Point.groovy @@ -0,0 +1,27 @@ +package com.fangjia.script.gj.location + +class Point { + Double lat; + Double lng; + + public Point(Double lng, Double lat){ + this.lat = lat + this.lng = lng + } + + Double getLat() { + return lat + } + + void setLat(Double lat) { + this.lat = lat + } + + Double getLng() { + return lng + } + + void setLng(Double lng) { + this.lng = lng + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/PointUtil.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/PointUtil.groovy new file mode 100644 index 000000000..ed710f2ee --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/PointUtil.groovy @@ -0,0 +1,143 @@ +package com.fangjia.script.gj.location + +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +import java.awt.geom.Point2D + +class PointUtil { + + public static final DBCollection BUSINESS_RANGE_COLL = MongoUtils.getCollectionByName("fangjia_crawl", "price_border", "mongo.db.fangjia.com") + + /** + * 判断当前位置是否在多边形区域内 + * + * @param checkPoint 当前经纬度 + * @param areaAround 围栏范围 + * @return + */ + public static boolean checkIsInPolygon(Point checkPoint, List areaAround) { + Point2D.Double point = new Point2D.Double(checkPoint.getLat(), checkPoint.getLng()); + List pointList = new ArrayList<>(); + for (Point everyPoint : areaAround) { + Point2D.Double polygonPoint = new Point2D.Double(everyPoint.getLat(), everyPoint.getLng()); + pointList.add(polygonPoint); + } + return checkIsPtInPoly(point, pointList); + } + + /** + * 判断点是否在多边形内,如果点位于多边形的顶点或边上,也算做点在多边形内,直接返回true + * + * @param point 检测点 + * @param pts 多边形的顶点 + * @return 点在多边形内返回true, 否则返回false + */ + public static boolean checkIsPtInPoly(Point2D.Double point, List pts) { + int N = pts.size(); + //如果点位于多边形的顶点或边上,也算做点在多边形内,直接返回true + boolean boundOrVertex = true; + //cross points count of x + int intersectCount = 0; + //浮点类型计算时候与0比较时候的容差 + double precision = 2e-10; + //neighbour bound vertices + Point2D.Double p1, p2; + //当前点 + Point2D.Double p = point; + //left vertex + p1 = pts.get(0); + //check all rays + for (int i = 1; i <= N; ++i) { + if (p.equals(p1)) { + //p is an vertex + return boundOrVertex; + } + //right vertex + p2 = pts.get(i % N); + //ray is outside of our interests + if (p.x < Math.min(p1.x, p2.x) || p.x > Math.max(p1.x, p2.x)) { + p1 = p2; + //next ray left point + continue; + } + //ray is crossing over by the algorithm (common part of) + if (p.x > Math.min(p1.x, p2.x) && p.x < Math.max(p1.x, p2.x)) { + //x is before of ray + if (p.y <= Math.max(p1.y, p2.y)) { + //overlies on a horizontal ray + if (p1.x == p2.x && p.y >= Math.min(p1.y, p2.y)) { + return boundOrVertex; + } + //ray is vertical + if (p1.y == p2.y) { + //overlies on a vertical ray + if (p1.y == p.y) { + return boundOrVertex; + //before ray + } else { + ++intersectCount; + } + } else { + //cross point on the left side + //cross point of y + double xinters = (p.x - p1.x) * (p2.y - p1.y) / (p2.x - p1.x) + p1.y; + //overlies on a ray + if (Math.abs(p.y - xinters) < precision) { + return boundOrVertex; + } + //before ray + if (p.y < xinters) { + ++intersectCount; + } + } + } + } else { + //special case when ray is crossing through the vertex + //p crossing over p2 + if (p.x == p2.x && p.y <= p2.y) { + //next vertex + Point2D.Double p3 = pts.get((i + 1) % N); + //p.x lies between p1.x & p3.x + if (p.x >= Math.min(p1.x, p3.x) && p.x <= Math.max(p1.x, p3.x)) { + ++intersectCount; + } else { + intersectCount += 2; + } + } + } + //next ray left point + p1 = p2; + } + //偶数在多边形外 + if (intersectCount % 2 == 0) { + return false; + } else { + //奇数在多边形内 + return true; + } + } + + public static void main(def a){ + new PointUtil().run() + } + def run(){ + def q = new BasicDBObject() + q.put("name", "淞南") + def obj = BUSINESS_RANGE_COLL.findOne(q) + if(obj){ + def pointStr = obj.border + if(pointStr){ + def pointArrs = pointStr.split(";") + def list = [] + for(def s : pointArrs){ + def arr = s.split(",") + list << new Point(arr[0].toDouble(), arr[1].toDouble()) + } + + println checkIsInPolygon(new Point(121.491478,31.327947), list) + } + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy index 50ffcf9dc..5be06fa7f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy @@ -60,12 +60,12 @@ class CrawlDistrictIdSelect { def findCrawlData(){ def q = new BasicDBObject() -// q.put("city","福州") -// q.put("source", "安居客") + q.put("city","福州") + q.put("source", "城市房产") q.put("s_date",202012) def dbcursor = thirdPartyColl.find(q).batchSize(10000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT) def retMap = [:] - def prefix = "anjuke.com/" + def prefix = ".cityhouse.cn/ha/" def keyList =new HashSet() for(def db : dbcursor){ def url = null @@ -77,11 +77,11 @@ class CrawlDistrictIdSelect { url = db.url } if(url && url.contains(prefix) ){ - def district_id = db.district_id//url.substring(url.indexOf(prefix)+prefix.length(),url.size()-1) + def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.indexOf(".html")) if(district_id=="0"){ continue } - def shortCity = url.substring(url.indexOf("//")+2, url.indexOf(".anjuke.com")) + def shortCity = url.substring(url.indexOf("//")+2, url.indexOf(prefix)) def key = shortCity +"#"+district_id def ids = retMap.get(key) @@ -92,7 +92,7 @@ class CrawlDistrictIdSelect { if(!keyList.contains(key)){ def map = new BasicDBObject() - map.root = "anjuke.com" + map.root = "cityhouse.com" map.city_short= shortCity map.city = db.city map.region = db.region diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy index 3dbb1a3e2..aa345e412 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy @@ -1,7 +1,35 @@ package com.fangjia.script.gj.test +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.Mongo + import java.text.SimpleDateFormat -SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); -Date date = new Date(1617211269868); -print dateFormat.format(date); \ No newline at end of file +//SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); +//Date date = new Date(1617211269868); +//print dateFormat.format(date); +DBCollection CITY_INFO_CONN = new Mongo("mongo.db.fangjia.com", 27017).getDB('fangjia_data_ana').getCollection('city_info'); +def path = "D:\\数据\\广发\\广东省小区及均价覆盖率统计.xlsx" +ExcelUtils.modify(path,{d-> + def districts = CommonDatautils.findAllDistrictByCity(d['B'], "district", ["name"]) + d['G'] = districts.size() + + def q = new BasicDBObject() + q.put("city", d['B']) + q.put("weekend", DateUtils.parseFromTightYYYYMMDD("2021-08-08")) + def obj = CITY_INFO_CONN.findOne(q); + if(obj){ + d['H'] = obj.price_coverage*100 + } + + +}) + + + + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index 93370a4a3..bbd42ab13 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -1,6 +1,7 @@ package com.fangjia.script.thirdPriceCrawl import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils import com.mongodb.BasicDBObject import org.apache.solr.common.util.Hash @@ -13,8 +14,9 @@ class SynchDistrictPrice { def static final SHORT_DATE = Utils.getLastMonth() def thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); def districtDetailColl = CrawalDBUtil.getDistrictDetailColl(); - def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房", - "anjuke.com":"安居客"] +// def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房", +// "anjuke.com":"安居客","cityhouse.cn":"城市房产"] + def sourceMap = ["zhuge.com":"诸葛找房"] static main(args) { def task = new SynchDistrictPrice() task.run() @@ -39,6 +41,7 @@ class SynchDistrictPrice { def q = new BasicDBObject() q.put("city", city) q.put("root", root) + q.put('m_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2021-07-01"))); def keys = new BasicDBObject() keys.put("city", 1) keys.put("region", 1) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DateUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DateUtils.groovy index a669411f2..bfa7f8ec7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DateUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DateUtils.groovy @@ -921,13 +921,13 @@ class DateUtils { public static String getRandomDate(def startDate, def endDate){ Date randomDate = randomDate(startDate, endDate); - SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss"); + SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd 00:00:00"); String result = format.format(randomDate); return result; } static main(args){ - + println getRandomDate("2021-08-01","2021-08-31") // nowFirstDay(); // nowLastDay(); /*def weekend = DateUtil.formatByPattern(getFirstDayOfWeek(new Date()), "yyyy-MM-dd"); @@ -955,8 +955,9 @@ class DateUtils { // Date randomDate = randomDate("2010-09-20", "2011-09-22"); // SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss"); // String result = format.format(randomDate); - System.out.println(getRandomDate("2010-09-20", "2011-09-22")); +// System.out.println(getRandomDate("2010-09-20", "2011-09-22")); + println DateUtils.getDayOfWeek(new Date()) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/RegionTransfer.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/RegionTransfer.groovy index 030871824..b4d6580d9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/RegionTransfer.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/RegionTransfer.groovy @@ -8,10 +8,10 @@ import static com.mongodb.client.model.Filters.* import com.fangjia.core.util.Mongo3Utils -def city = "莱芜" -def destCity = "济南" -def region = "" -def destRegion = "莱芜" +def city = "洛阳" +def destCity = "洛阳" +def region = "吉利" +def destRegion = "孟津"