From 5d8e2bb9435a1711b52c8d64973829d28613076f Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 22 Nov 2023 10:27:01 +0800 Subject: [PATCH 1/2] update --- .../script/commData/CommonDatautils.groovy | 15 ++++ .../fangjia/script/gj/abc/SuzouPocTest.groovy | 4 +- .../fangjia/script/gj/district/ApiTest.groovy | 41 +++++++---- .../gj/district/FangjiaClientUtils.groovy | 1 + .../gj/xiamen/SuggestionDataSynch.groovy | 71 +++++++++++++++---- .../com/fangjia/script/util/ExcelUtils.groovy | 2 +- 6 files changed, 106 insertions(+), 28 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 25102297b..c40d48b46 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -3,6 +3,7 @@ package com.fangjia.script.commData import com.fangjia.core.RegionData import com.fangjia.httpclient.UserAgents import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.looplineinfo.InsidePolygon import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.JacksonUtil @@ -90,6 +91,9 @@ class CommonDatautils { public static final DBCollection confirm_price_date = MongoUtils.getCollectionByName("fangjia", "confirm_price_data", "mongo.db.fangjia.com") + public static final DBCollection TOWN_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline") + + static def DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"] private static final HashMap> FITMENTS_TYPES_MAPS = new HashMap>() static{ @@ -4217,6 +4221,17 @@ class CommonDatautils { return price } + static def findTownRange(city){ + def q = new BasicDBObject() + q.put("city",city) + def dbcursor = TOWN_POLYLINECOLL.find(q) + def townMap = [:] + for(def d : dbcursor){ + def pointList = InsidePolygon.createPoint3(d.polyline); + townMap.put(d.fjw_region +"##"+ d.town, pointList) + } + return townMap + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/abc/SuzouPocTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/abc/SuzouPocTest.groovy index 001f9cea6..f1ca243e2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/abc/SuzouPocTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/abc/SuzouPocTest.groovy @@ -23,7 +23,7 @@ class SuzouPocTest { } def run(){ - def path = "D:\\数据\\江苏农行\\江苏农1062条.xlsx" + def path = "D:\\数据\\江苏农行\\江苏农1062条(合).xlsx" addressEvalute(path) // addFjwTobPrice(path) // addThirdPrice(path) @@ -33,7 +33,7 @@ class SuzouPocTest { def addressEvalute(path){ - ExcelUtils.modify(path,{d-> + ExcelUtils.multiThreadingModify(path,{d-> if(!d['O'] && d['H']){ def city = d['D'] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ApiTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ApiTest.groovy index bf20894f3..bd28d23a2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ApiTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ApiTest.groovy @@ -8,6 +8,7 @@ import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.OpenApiClient import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import net.sf.json.JSONObject import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.CountDownLatch @@ -41,21 +42,37 @@ class ApiTest { } def addrStructure(path){ - ExcelUtils.modify(path,{d-> + ExcelUtils.multiThreadingModify(path,{d-> if(d['B']){ - def jsonObject = FangjiaClientUtils.formatFivelevel(d['A'], d['B']); - if(jsonObject && jsonObject.containsKey("result")) { - d['C'] = jsonObject.get("province") - d['D'] = jsonObject.get("city") - d['E'] = jsonObject.get("district") - d['F'] = jsonObject.get("township") - d['G'] = jsonObject.get("address") - d['H'] = jsonObject.get("provinceCode") - d['I'] = jsonObject.get("cityCode") - d['J'] = jsonObject.get("districtCode") + JSONObject jsonObject = FangjiaClientUtils.formatFivelevel(d['A'], d['B']); + if(jsonObject) { + if(jsonObject.get("province") && jsonObject.get("province")!="null"){ + d['C'] = jsonObject.get("province") + } + if(jsonObject.get("city") && jsonObject.get("city")!="null"){ + d['D'] = jsonObject.get("city") + } + if(jsonObject.get("district") && jsonObject.get("district")!="null"){ + d['E'] = jsonObject.get("district") + } + if(jsonObject.get("township") && jsonObject.get("township")!="null"){ + d['F'] = jsonObject.get("township") + } + if(jsonObject.get("address") && jsonObject.get("address")!="null"){ + d['G'] = jsonObject.get("address") + } + if(jsonObject.get("provinceCode") && jsonObject.get("provinceCode")!="null"){ + d['H'] = jsonObject.get("provinceCode") + } + if(jsonObject.get("cityCode") && jsonObject.get("cityCode")!="null"){ + d['I'] = jsonObject.get("cityCode") + } + if(jsonObject.get("districtCode") && jsonObject.get("districtCode")!="null"){ + d['J'] = jsonObject.get("districtCode") + } } } - },0); + },4); } def addressEvaluate(path){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index b826a7dd2..1ccbb15b5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -319,6 +319,7 @@ class FangjiaClientUtils { } content = new HttpRequest().newGet(url, null, "utf-8") jsonObj = JSONObject.fromObject(content) + println jsonObj } catch (Exception e) { println address +" "+content e.printStackTrace(); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SuggestionDataSynch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SuggestionDataSynch.groovy index c25d2360a..7b1dd5485 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SuggestionDataSynch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SuggestionDataSynch.groovy @@ -4,9 +4,11 @@ import com.fangjia.core.RegionData import com.fangjia.script.BaseBatch import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.looplineinfo.InsidePolygon import com.fangjia.script.util.DateUtils import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.ProvinceData +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject @@ -44,36 +46,36 @@ class SuggestionDataSynch extends BaseBatch{ @Override public void process(Object args) { -// run() - updateCode() + run() +// updateCode() } def run() { //"三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆" - def citys = CommonDatautils.findAllCity() + def citys = CommonDatautils.findAllCity().reverse() def index = 0 fullCityMap = queryFullNameCity() mcaDistMap = queryDistNameCity() for (def c : citys) { index++ - if(!["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"].contains(c)){ +// if(!["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"].contains(c)){ updating(c, "district") updating(c, "office") updating(c, "shop") println index + " " + c + " 完成" - +// } + if(["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"].contains(c)) { + poiData(c) } -// poiData(c) - } } def updating(city, cat){ - def dists = CommonDatautils.findAllDistrictsFromMg(city, cat, ["city","region","name","address","mapping_alias","addr_alias","alias","cat","estate_type2","lng","lat"]) + def dists = CommonDatautils.findAllDistrictsFromMg(city, cat, ["city","region","name","address","mapping_alias","addr_alias","alias","cat","estate_type2","lng","lat","township"]) def rets = new CopyOnWriteArrayList() def totalCount = new AtomicLong() @@ -143,6 +145,7 @@ class SuggestionDataSynch extends BaseBatch{ map.location = loc } map.source = "FJW" + map.township = dist.township return map } @@ -171,17 +174,18 @@ class SuggestionDataSynch extends BaseBatch{ def poiData(city){ + def townMap = CommonDatautils.findTownRange(city) def data = fullCityMap.get(city) if(data){ def results = CommonDatautils.findAmapDetailBySql(data.city) - createPoi(results, data) + createPoi(results, data, townMap) println city +" 完成!" } } - def createPoi(results, data){ + def createPoi(results, data, townMap){ def rets = new CopyOnWriteArrayList() def totalCount = new AtomicLong() @@ -208,6 +212,15 @@ class SuggestionDataSynch extends BaseBatch{ map.cat = dist.cat map.location = dist.location map.source = "FJW" + + map.township = InsidePolygon.isInArea2(map.location.lon, map.location.lat, townMap) + if(!map.township){ + def gaodeMap = getTownshipInfo(map.location.lon+","+map.location.lat) + if(gaodeMap && gaodeMap.township){ + map.township = gaodeMap.township + } + } + println map.township //往100 上同步 updateData(map) println dist.city +" "+totalCount.get() +"/"+results.size()+" "+dist.name+" "+map.id @@ -242,9 +255,9 @@ class SuggestionDataSynch extends BaseBatch{ } } } - }else{ - if(d.city == "省直辖县级行政区划" && d.district){ - fjwcity = RegionData.formatCity(d.district) + }else if(d.level == "区域"){ + if(d.administer == "省直辖县级行政区划"){ + fjwcity = RegionData.formatCity(d.city) if(fjwcity){ d.city = d.district } @@ -305,4 +318,36 @@ class SuggestionDataSynch extends BaseBatch{ administrative_division_100_CONN.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata)); } } + + def getTownshipInfo(def glocation){ + def township = null + def district = null + def jsonObject = null + def stop = false + def count = 0 + while(!stop){ + if(glocation){ + jsonObject = GaoDeApiUtils.geocode(glocation) + if(jsonObject && jsonObject.regeocode) { + def addressComponent = jsonObject.regeocode.addressComponent + if(addressComponent){ + township = addressComponent.township + district = addressComponent.district + } + } + } + if(district || count>10){ + stop = true + }else{ + count ++ + Thread.sleep(3000) + print "." + } + } + + def map = [:] + map.district = district + map.township = township + return map; + } } \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy index 85d25ddaf..4d263c58e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy @@ -414,7 +414,7 @@ public class ExcelUtils{ def data = ExcelUtils.read(fileName, srcSheetIndex, 0) def body = data.subList(1, data.size()) - def nThread = Runtime.getRuntime().availableProcessors() * 5 + def nThread = Runtime.getRuntime().availableProcessors() * 3 println nThread ExecutorService executorService = Executors.newFixedThreadPool(nThread) final AtomicInteger counter = new AtomicInteger(0) From bdfa62a42f7364358bacb0e555e79fbaccb547ee Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 22 Nov 2023 10:32:01 +0800 Subject: [PATCH 2/2] up --- .../com/fangjia/script/gj/xiamen/SuggestionDataSynch.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SuggestionDataSynch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SuggestionDataSynch.groovy index 7b1dd5485..4362615db 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SuggestionDataSynch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SuggestionDataSynch.groovy @@ -54,7 +54,7 @@ class SuggestionDataSynch extends BaseBatch{ def run() { //"三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆" - def citys = CommonDatautils.findAllCity().reverse() + def citys = CommonDatautils.findAllCity() def index = 0 fullCityMap = queryFullNameCity() mcaDistMap = queryDistNameCity()