From 6c769c15ff1fd7f34c301d00a5a131c25d7d6cc5 Mon Sep 17 00:00:00 2001 From: mjt <13607600062@163.com> Date: Mon, 29 Jan 2024 15:34:21 +0800 Subject: [PATCH] =?UTF-8?q?suggestion=20=E5=A2=9E=E5=8A=A0=E5=88=A4?= =?UTF-8?q?=E6=96=AD=E5=B7=B2=E7=BB=8F=E5=A4=84=E7=90=86=E8=BF=87=E7=9A=84?= =?UTF-8?q?=E5=9F=8E=E5=B8=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gj/xiamen/SuggestionDataSynch.groovy | 152 +++++++++++++++--- 1 file changed, 130 insertions(+), 22 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SuggestionDataSynch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SuggestionDataSynch.groovy index 4362615db..86eac3172 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SuggestionDataSynch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SuggestionDataSynch.groovy @@ -4,6 +4,7 @@ import com.fangjia.core.RegionData import com.fangjia.script.BaseBatch import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.lixue.DB.City import com.fangjia.script.looplineinfo.InsidePolygon import com.fangjia.script.util.DateUtils import com.fangjia.script.util.MongoUtils @@ -20,13 +21,15 @@ import java.util.concurrent.atomic.AtomicLong class SuggestionDataSynch extends BaseBatch{ static def targetUrl = "http://192.168.0.12:9200/seaweed/" - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50) def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL) DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL) DBCollection SUGGESTION_CONN = MongoUtils.getCollectionByName2("fangjia", "suggestion", MONGO100_URL) + DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MONGO100_URL) + public static def provinceMap = CommonDatautils.findAllCityProvince() def lastMonthDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); @@ -42,14 +45,31 @@ class SuggestionDataSynch extends BaseBatch{ static main(args) { runBatch(new SuggestionDataSynch(), args); + } @Override public void process(Object args) { run() // updateCode() +// getExitsCity() + } + def getExitsCity(){ + SUGGESTION_CONN = MongoUtils.getCollectionByName2("fangjia", "suggestion", MONGO100_URL) + def cities = SUGGESTION_CONN.distinct("city") + def simpleCities = [] + for (def c :cities) { + def city = RegionData.formatCity(c) + simpleCities.add(city) + } +// print(simpleCities) + return simpleCities + } + + + def run() { @@ -58,22 +78,74 @@ class SuggestionDataSynch extends BaseBatch{ def index = 0 fullCityMap = queryFullNameCity() mcaDistMap = queryDistNameCity() + def existCities = getExitsCity() for (def c : citys) { + + def allRets = [] + +// if(["上海","北京","广州","深圳","成都","杭州","天津","西安","南京","沈阳","重庆","武汉","石家庄","福州","宁波","贵阳","怀化","郴州","益阳","永州","娄底","湘西","长沙","常德","株洲","岳阳","衡阳","湘潭","张家界","邵阳","南京","苏州","无锡","扬州","常州","徐州","镇江","南通","泰州","连云港","盐城","淮安","宿迁"].contains(c)){ + if(existCities.contains(c)){ + continue; + } index++ // if(!["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"].contains(c)){ - updating(c, "district") - updating(c, "office") - updating(c, "shop") - println index + " " + c + " 完成" -// } - if(["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"].contains(c)) { - poiData(c) - } + def ret1 = updating(c, "district") + def ret2 = updating(c, "office") + def ret3 = updating(c, "shop") +// } +// if(["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"].contains(c)) { + def ret4 = poiData(c) +// } + + allRets.addAll(ret1) + allRets.addAll(ret2) + allRets.addAll(ret3) + allRets.addAll(ret4) + + saveToMG(c, allRets) + println index + " " + c + " 完成 "+ allRets.size() } } + + + def saveToMG(city, rows){ + + def data = fullCityMap.get(city) + def q = new BasicDBObject("city", data.city) + def count = 0 + try{ + count = SUGGESTION_CONN.count(q) + }catch(e){ + SUGGESTION_CONN = MongoUtils.getCollectionByName2("fangjia", "suggestion", MONGO100_URL) + count = SUGGESTION_CONN.count(q) + } +// if(count>0){ +// SUGGESTION_CONN.remove(q) +// } + + def rets = [] + for(def i=0;i + obj.put(k,v) + } + rets << obj + if(i%50000==0){ + SUGGESTION_CONN.insert(rets) + rets.clear() + } + } + if(rets.size()>0){ + SUGGESTION_CONN.insert(rets) + rets.clear() + } + } + + def updating(city, cat){ def dists = CommonDatautils.findAllDistrictsFromMg(city, cat, ["city","region","name","address","mapping_alias","addr_alias","alias","cat","estate_type2","lng","lat","township"]) def rets = new CopyOnWriteArrayList() @@ -93,7 +165,8 @@ class SuggestionDataSynch extends BaseBatch{ def map = createData(dist) //往100 上同步 - updateData(map) +// updateData(map) + rets << map println city +" "+cat +" "+totalCount.get() +"/"+dists.size()+" "+dist.name+" "+map.id } } catch (Exception e) { @@ -135,7 +208,7 @@ class SuggestionDataSynch extends BaseBatch{ map.alias = dist.alias } if(dist.addr_alias && dist.addr_alias.size()>0){ - map.addr_alias = dist.addr_alias + map.addr_alias = dist.addr_alias } map.type = dist.cat if(dist.lng){ @@ -178,17 +251,19 @@ class SuggestionDataSynch extends BaseBatch{ def data = fullCityMap.get(city) if(data){ def results = CommonDatautils.findAmapDetailBySql(data.city) - createPoi(results, data, townMap) - println city +" 完成!" + def rets = createPoi(data.city, results, data, townMap) + return rets; } + return [] } - def createPoi(results, data, townMap){ + def createPoi(fullCity, results, data, townMap){ def rets = new CopyOnWriteArrayList() def totalCount = new AtomicLong() + def townshipMap = findMapPoiTownshipMap(fullCity) final CountDownLatch latch = new CountDownLatch(results.size()) for(def i=0;i