From 9c744bb3bead62eca4610d66e7769bb6f2871952 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 8 Jan 2024 18:23:01 +0800 Subject: [PATCH] update --- .../script/gj/village/govLocation.groovy | 110 ++++++++++++++++++ .../gj/xiamen/SuggestionDataSynch.groovy | 65 +++++++++-- 2 files changed, 163 insertions(+), 12 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/govLocation.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/govLocation.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/govLocation.groovy new file mode 100644 index 000000000..d0877a9a0 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/govLocation.groovy @@ -0,0 +1,110 @@ +package com.fangjia.script.gj.village + +import com.fangjia.script.gj.EsSearchUtils +import com.fangjia.script.looplineinfo.InsidePolygon +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +DBCollection TOWN_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline") + +def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; +DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL) + +def filePath = "D:\\数据\\涉农\\县.xlsx" +ExcelUtils.multiThreadingModify(filePath, { row -> +// if(!row['D']){ +// def rets = EsSearchUtils.findPois(row['B'], row['C'], row['C']+"人民政府") +// if(rets){ +// def obj = findOneByName(rets, row['C']+"人民政府") +// if(obj){ +// row['D'] = obj.lng +// row['E'] = obj.lat +// println row['C'] + " "+obj +// } +// } +// } + +// if(row['D'] && !row['F']){ +// def townMap = findTownRange(row['B'], TOWN_POLYLINECOLL) +// def map = ["lon":row['D'], "lat":row['E']] +// def township = InsidePolygon.isInArea2(map.lon, map.lat, townMap) +// row['F'] = township +// } + + if(row['F'] && !row['K']){ + def q = new BasicDBObject() + q.put("city_name", row['B']) + q.put("area_name", row['C']) + def streetName = row['I'] + q.put("street_name", streetName) + def count = COMMUNITY_CONN.count(q); + row['K'] = count +// row['H'] = streetName + } +// def gaodeMap = getTownshipInfo(row['D']+","+row['E']) +// if(gaodeMap && gaodeMap.township){ +// row['I'] = gaodeMap.township +// } + +}) + + +def getTownshipInfo(def glocation){ + def township = null + def district = null + def jsonObject = null + def stop = false + def count = 0 + while(!stop){ + if(glocation){ + jsonObject = GaoDeApiUtils.geocode(glocation) + if(jsonObject && jsonObject.regeocode) { + def addressComponent = jsonObject.regeocode.addressComponent + if(addressComponent){ + township = addressComponent.township + district = addressComponent.district + } + } + } + if(district || count>10){ + stop = true + }else{ + count ++ + Thread.sleep(3000) + print "." + } + } + + def map = [:] + map.district = district + map.township = township + return map; +} + + +def findOneByName(def rets, name){ + def obj = null + for(def r : rets){ + if(r.name == name){ + obj = r + break; + } + } + return obj +} + + +def findTownRange(city, TOWN_POLYLINECOLL){ + def q = new BasicDBObject() + q.put("city",city) + def dbcursor = TOWN_POLYLINECOLL.find(q) + def townMap = [:] + for(def d : dbcursor){ + def pointList = InsidePolygon.createPoint3(d.polyline); + townMap.put(d.fjw_region +"##"+ d.town, pointList) + } + return townMap +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SuggestionDataSynch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SuggestionDataSynch.groovy index 104e28def..b9070a74a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SuggestionDataSynch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SuggestionDataSynch.groovy @@ -63,22 +63,59 @@ class SuggestionDataSynch extends BaseBatch{ mcaDistMap = queryDistNameCity() for (def c : citys) { + def allRets = [] index++ // if(!["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"].contains(c)){ - updating(c, "district") - updating(c, "office") - updating(c, "shop") + def ret1 = updating(c, "district") + def ret2 = updating(c, "office") + def ret3 = updating(c, "shop") // } // if(["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"].contains(c)) { - poiData(c) + def ret4 = poiData(c) // } - println index + " " + c + " 完成" + allRets.addAll(ret1) + allRets.addAll(ret2) + allRets.addAll(ret3) + allRets.addAll(ret4) + println index + " " + c + " 完成 "+ allRets.size() + saveToMG(c, allRets) } } + + + def saveToMG(city, rows){ + + def data = fullCityMap.get(city) + def q = new BasicDBObject("city", data.city) + def count = SUGGESTION_CONN.count(q) + if(count>0){ + SUGGESTION_CONN.remove(q) + } + + def rets = [] + for(def i=0;i + obj.put(k,v) + } + rets << obj + if(i%50000==0){ + SUGGESTION_CONN.insert(rets) + rets.clear() + } + } + if(rets.size()>0){ + SUGGESTION_CONN.insert(rets) + rets.clear() + } + } + + def updating(city, cat){ def dists = CommonDatautils.findAllDistrictsFromMg(city, cat, ["city","region","name","address","mapping_alias","addr_alias","alias","cat","estate_type2","lng","lat","township"]) def rets = new CopyOnWriteArrayList() @@ -98,7 +135,8 @@ class SuggestionDataSynch extends BaseBatch{ def map = createData(dist) //往100 上同步 - updateData(map) +// updateData(map) + rets << map println city +" "+cat +" "+totalCount.get() +"/"+dists.size()+" "+dist.name+" "+map.id } } catch (Exception e) { @@ -140,7 +178,7 @@ class SuggestionDataSynch extends BaseBatch{ map.alias = dist.alias } if(dist.addr_alias && dist.addr_alias.size()>0){ - map.addr_alias = dist.addr_alias + map.addr_alias = dist.addr_alias } map.type = dist.cat if(dist.lng){ @@ -183,9 +221,11 @@ class SuggestionDataSynch extends BaseBatch{ def data = fullCityMap.get(city) if(data){ def results = CommonDatautils.findAmapDetailBySql(data.city) - createPoi(results, data, townMap) + def rets = createPoi(results, data, townMap) println city +" 完成!" + return rets; } + return [] } @@ -227,7 +267,8 @@ class SuggestionDataSynch extends BaseBatch{ } println map.township //往100 上同步 - updateData(map) +// updateData(map) + rets << map println dist.city +" "+totalCount.get() +"/"+results.size()+" "+dist.name+" "+map.id } } catch (Exception e) { @@ -256,13 +297,13 @@ class SuggestionDataSynch extends BaseBatch{ if(d.city == "市辖区"){ fjwcity = RegionData.formatCity(d.province) if(fjwcity){ - d.city = d.province + d.city = d.province } } } }else if(d.level == "区域"){ if(d.administer == "省直辖县级行政区划"){ - fjwcity = RegionData.formatCity(d.city) + fjwcity = RegionData.formatCity(d.city) if(fjwcity){ d.city = d.district } @@ -290,7 +331,7 @@ class SuggestionDataSynch extends BaseBatch{ }else if(d.city == "市辖区"){ def fjwcity = RegionData.formatCity(d.province) if(fjwcity){ - def fjwdist = RegionData.formatRegion(fjwcity, d.district) + def fjwdist = RegionData.formatRegion(fjwcity, d.district) retMap.put(fjwcity+"#"+fjwdist, d) } }else{