From 9e23071ef58e6d582d1780c72ee7942071da1d48 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 9 Oct 2023 17:56:59 +0800 Subject: [PATCH] update --- .../script/ccbevaluation/TradeStatLog.groovy | 8 +- .../gj/xiamen/GaoDePoiDataProcess.groovy | 90 ++++++--- .../fangjia/script/gj/xiamen/PoiWash.groovy | 174 ++++++++++++++++-- 3 files changed, 227 insertions(+), 45 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index 4f338e082..df0d2d927 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -40,8 +40,8 @@ class TradeStatLog { static main(args) { def task = new TradeStatLog() - task.run()//统计最近3个月抓取的成交数据量 -// task.exportTrade()//导出未入库成交 +// task.run()//统计最近3个月抓取的成交数据量 + task.exportTrade()//导出未入库成交 // task.thirdPartyStat() // println Calendar.getInstance().get(Calendar.YEAR) // task.averageStat() @@ -499,8 +499,8 @@ class TradeStatLog { def headers = ["ID", "城市", "区域", "名称", "来源","价格", "面积","地址", "成交时间","过滤原因"] def rets = [] rets << headers - def startDate = "2022-03-01" - def endDate = "2022-04-01" + def startDate = "2023-08-01" + def endDate = "2023-10-01" for(def city : citys){ def trades = findTradesDetail(city, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd")) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy index 55100de2c..9d030c077 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy @@ -11,6 +11,7 @@ import com.fangjia.script.looplineinfo.InsidePolygon import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils +import com.fangjia.script.util.ProvinceData import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection @@ -36,7 +37,7 @@ import java.util.concurrent.atomic.AtomicLong @BatchOwner(Watcher.GaoJuan) class GaoDePoiDataProcess extends BaseBatch{ - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) public static def version = new Date().getTime() @@ -47,7 +48,7 @@ class GaoDePoiDataProcess extends BaseBatch{ def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; - DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MAP_POI_URL) + DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi_all_8", MAP_POI_URL) static DBCollection TOWN_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline") @@ -59,11 +60,12 @@ class GaoDePoiDataProcess extends BaseBatch{ @Override void process(Object args) { -// synchMapPoi(); +// println ProvinceData.getProvince("成都") + synchMapPoi(); // synchCompany() // poiCatProcess() // addressAndCatProcess() - synchNetwork() +// synchNetwork() } def synchCompany(){ @@ -91,22 +93,26 @@ class GaoDePoiDataProcess extends BaseBatch{ def synchMapPoi(){ //"三明市","漳州市","厦门市","福州市" //三明市,"宁德市","泉州市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市","重庆市" - def citys = ["福州市","重庆市"] + def citys = ["上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市"] for(def city : citys){ + def townMap = findTownRange(city) def results = CommonDatautils.findAmapDetailBySql(city) - println results.size() - def rows = running(results, "map") - save(city, rows) + println "city: "+city +" town: "+townMap.size()+" poi:"+results.size() + def rows = running(city, results, "map", townMap) +// save(city, rows) + saveToMG(city, rows) } } - def running(results, type, townMap){ + def running(city,results, type, townMap){ def rets = new CopyOnWriteArrayList() def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(results.size()) + def prov = ProvinceData.getProvince(city.replace("市","")) + for(def i=0;i=3){ map.small_type = arrs[2] } + if(arrs.size()>3){ + map.cat = arrs[0]+";"+arrs[1]+";"+arrs[2] + } } + + map.township = InsidePolygon.isInArea2(map.location.lon, map.location.lat, townMap) + //高德转百度 def baiduMap = GeoUtils.gcj02ToBd09(map.location.lat, map.location.lon) if(baiduMap){ @@ -251,18 +258,24 @@ class GaoDePoiDataProcess extends BaseBatch{ map.gps_location = gpslocation } - def gaodeMap = getTownshipInfo(map.location.lon+","+map.location.lat) - if(gaodeMap && gaodeMap.township){ - map.township = gaodeMap.township - } + if(!map.township){ - if(!fjw_region){ - if(gaodeMap && gaodeMap.district){ - fjw_region = RegionData.formatRegion(fjw_city, gaodeMap.district) + def gaodeMap = getTownshipInfo(map.location.lon+","+map.location.lat) + if(gaodeMap && gaodeMap.township){ + map.township = gaodeMap.township } - } - if(!fjw_region){ - println fjw_city+" "+map.region+" 无法格式化 "+map.location + " "+gaodeMap + + if(!fjw_region){ + if(gaodeMap && gaodeMap.district){ + fjw_region = RegionData.formatRegion(fjw_city, gaodeMap.district) + } + } + + if(!fjw_region){ + println fjw_city+" "+map.region+" 无法格式化 "+map.location + " "+gaodeMap + } + + map.fjw_district = fjw_region } map.fjw_district = fjw_region map.spell_name = com.fangjia.core.util.DataUtil.hanyuQuanpin(map.name, " ") @@ -320,6 +333,27 @@ class GaoDePoiDataProcess extends BaseBatch{ } + def saveToMG(city, rows){ + + def rets = [] + for(def i=0;i + obj.put(k,v) + } + rets << obj + if(i%50000==0){ + MAPPOI_CONN.insert(rets) + rets.clear() + } + } + if(rets.size()>0){ + MAPPOI_CONN.insert(rets) + rets.clear() + } + } + def getTownshipInfo(def glocation){ def township = null def district = null diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/PoiWash.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/PoiWash.groovy index b4266d748..1b008b4b4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/PoiWash.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/PoiWash.groovy @@ -1,6 +1,9 @@ package com.fangjia.script.gj.xiamen +import com.fangjia.core.RegionData +import com.fangjia.core.util.DataUtil import com.fangjia.script.util.MongoUtils +import com.fangjia.script.xingyebank.Data import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.rabbitmq.client.impl.AMQImpl @@ -14,31 +17,42 @@ import java.util.concurrent.atomic.AtomicLong class PoiWash { def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; - DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi_test", MAP_POI_URL) + DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi_all", MAP_POI_URL) DBCollection TOWNSHIP_CONN = MongoUtils.getCollectionByName2("fangjia", "township", MAP_POI_URL) - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) public static def townsMap = [:] + public static def regions = [] static main(args) { def task = new PoiWash() task.run() +// task.check() + +// def s = "002乡道与767县道交叉口东北约60米 " +// def s1 = DataUtil.toDBCcase(s) +// println s1 + } def run(){ - - //"三明市", - def citys = ["宁德市","泉州市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市","重庆市"] + //"上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市" + //"重庆市","三明市","宁德市","泉州市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市" + def citys = ["泉州市", "三明市","宁德市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市","重庆市"] for(def city : citys){ townsMap = findAllTownship(city) + regions = findAllRegions(city) def datas = findPois(city) - println city +' '+datas.size() + println city +' poi: '+datas.size()+" towns:"+ townsMap.get(RegionData.formatCity(city)).size() process(city, datas) +// updateOldRegion(city) println city +" 完成" } } + + def process(city, results){ // def rets = new CopyOnWriteArrayList() def totalCount = new AtomicLong() @@ -70,7 +84,6 @@ class PoiWash { } latch.await() - // return rets; } @@ -85,10 +98,17 @@ class PoiWash { addr = addr.replace(data.city,'') } if(data.district){ - addr = addr.replace(data.district,'') + if(!addr.contains(data.district+"路")){ + addr = addr.replace(data.district,'') + } } if(data.township){ - addr = addr.replace(data.township,'') + try{ + addr = addr.replace(data.township,'') + }catch(Exception e){ + println data.id+" "+data.township + } + } if(addr && addr.startsWith('-')){ addr = addr.replace('---','').replace('--','').replace('-','') @@ -97,6 +117,12 @@ class PoiWash { return null; } + if(addr){ + for(def r : regions){ + addr = addr.replace(r, "") + } + } + if(addr){ def towns = townsMap.get(data.fjw_city) for(def t : towns){ @@ -115,9 +141,11 @@ class PoiWash { def findPois(def city){ def q = new BasicDBObject() q.put("city", city) +// q.put("name", '三明海大饲料有限公司') def keys = new BasicDBObject() keys.put("_id", 1) + keys.put("id", 1) keys.put("province", 1) keys.put("city", 1) keys.put("district", 1) @@ -125,7 +153,7 @@ class PoiWash { keys.put("address", 1) keys.put("fjw_city", 1) - def dbcursor = MAPPOI_CONN.find(q, keys) + def dbcursor = MAPPOI_CONN.find(q, keys).batchSize(10000) def rets = [] for(def d : dbcursor){ rets << d @@ -133,13 +161,78 @@ class PoiWash { return rets; } + + def updateOldRegion(def city){ + def q = new BasicDBObject() + q.put("city", city) + def keys = new BasicDBObject() + keys.put("id", 1) + keys.put("city", 1) + keys.put("district", 1) + keys.put("fjw_city", 1) + keys.put("fjw_district", 1) + keys.put("township", 1) + + def dbcursor = MAPPOI_CONN.find(q, keys).batchSize(10000) + def rets = [] + + def distMap = [:] + for(def d : dbcursor){ +// def region = distMap.get(d.fjw_district) +// if(!region){ +// distMap.put(d.fjw_district, d.district) +// } +// +// if(d.district instanceof Double){ +// def newRegion = distMap.get(d.fjw_district) +// if(newRegion){ +// println d.id +" "+d.fjw_district+" "+newRegion +// def udata = new BasicDBObject() +// udata.put("district", newRegion) +// MAPPOI_CONN.update(new BasicDBObject('_id',d._id),new BasicDBObject('$set', udata)) +// } +// } + + if(d.township instanceof Double){ + def udata = new BasicDBObject() + udata.put("township", "") + MAPPOI_CONN.update(new BasicDBObject('_id',d._id),new BasicDBObject('$set', udata)) + println d.id + } + } + return rets; + } + + def findAllRegions(cityName){ + def q = new BasicDBObject() + if(["重庆市","北京市","天津市", "上海市"].contains(cityName)){ + q.put("province_name", cityName) + }else{ + q.put("city_name", cityName) + } + def dbcursor = TOWNSHIP_CONN.find(q).batchSize(2000) + def set = new HashSet() + for(def d : dbcursor){ + set << d.area_name + } + return set + } + + def findAllTownship(cityName){ def q = new BasicDBObject() - q.put("city_name", cityName) - def dbcursor = TOWNSHIP_CONN.find(q) + if(["重庆市","北京市","天津市", "上海市"].contains(cityName)){ + q.put("province_name", cityName) + }else{ + q.put("city_name", cityName) + } + def dbcursor = TOWNSHIP_CONN.find(q).batchSize(2000) def map = [:] for(def d : dbcursor){ def city = d.fangjia_city + if(!city || city instanceof Double){ + city = RegionData.formatCity(d.city_name) + } def towns = map.get(city) if(!towns){ towns = [] @@ -150,9 +243,64 @@ class PoiWash { if(alias){ towns.addAll(alias) } - towns.unique() +// towns.unique() } return map } + + def check(){ + def citys = ["三明市","宁德市","泉州市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市","重庆市","三明市"] + for(def city : citys){ + def datas = findPois(city) + processUpdate(city, datas) + println city +" 完成" + } + } + + def processUpdate(city, results){ +// def rets = new CopyOnWriteArrayList() + def totalCount = new AtomicLong() + def updateTotalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(results.size()) + + def count = results.size() + for(def i=0;i