From 502c682c8bdf8bd9f45668a2dad08afd8ca14a03 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 25 Nov 2024 18:34:02 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E7=89=88poi=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/commData/CommonDatautils.groovy | 34 ++- .../gj/data_handle/NewGaoDePoiBatch.groovy | 232 ++++++++++++++++++ 2 files changed, 263 insertions(+), 3 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 4d1ea8f76..a1bc0bd49 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -4209,13 +4209,13 @@ class CommonDatautils { static def findAllNewPoiDatas(city){ - String sql = "select cityname,adname,name,address,location from poi_amap_2024 where cityname='${city}'" - def rets = [:] + String sql = "select pname,cityname,adname,name,address,location,type,id from poi_amap_2024 where cityname='${city}' limit 100" + def rets = [] def index= 0 ElasticUtils.executeSqlQuery(sql, { map-> index++ println index - rets << ["map_id":map.id,"city":map.cityname, "district":map.adname, "name":map.name, "address":map.address, "type":map.type,"lng":map.location.x, "lat":map.location.y] + rets << ["map_id":map.id,"province":map.pname,"city":map.cityname, "district":map.adname, "name":map.name, "address":map.address, "cat":map.type,"lng":map.location.x, "lat":map.location.y] }) return rets; } @@ -5149,6 +5149,34 @@ class CommonDatautils { } + static def queryFullRegionTownByCity(city, fields){ + + def q = new BasicDBObject() + q.put("type","乡镇街道") + q.put("city",city) + + def keys = new BasicDBObject() + for(def key : fields){ + keys.put(key,1) + } + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT) + def townMap = [:] + for(def d : dbcursor){ + if(d.district){ + def list = townMap.get(d.district) + if(!list){ + list = [] + } + list << d + townMap.put(d.district, list) + } + } + println "[${city}] 乡镇街道 加载完成 "+new Date().format("yyyy-MM-dd HH:mm:ss") + return townMap + } + + + static def queryTownByCity(city){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy new file mode 100644 index 000000000..b8621d41c --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy @@ -0,0 +1,232 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.xiamen.ElasticUtils89 +import com.fangjia.script.looplineinfo.InsidePolygon +import com.fangjia.script.util.ExcelUtils +import org.elasticsearch.action.bulk.BulkRequest +import org.elasticsearch.action.bulk.BulkResponse +import org.elasticsearch.action.index.IndexRequest +import org.elasticsearch.action.support.WriteRequest +import org.elasticsearch.client.RequestOptions +import org.elasticsearch.common.xcontent.XContentType + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong +/** + * 新版poi处理入库(第二批) + */ +@BatchOwner(Watcher.GaoJuan) +class NewGaoDePoiBatch extends BaseBatch{ + + public static def version = new Date().getTime() + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + + public static def TAGS = [] + + static main(args) { + runBatch(new NewGaoDePoiBatch(), args) + } + + @Override + void process(Object args) { + run() + } + + def run(){ + if (CommonDatautils.isWindow()) { + TAGS = getAllPoiType("D:\\数据\\福建省_POI_GCJ02_20240630\\POI分类白名单.xlsx") + }else{ + TAGS = getAllPoiType("/home/goojia/gj/POI_TYPES.xlsx") + } + + + def cityList = CommonDatautils.findFullNameCitys().reverse() + for(def cityObj : cityList){ + if(cityObj.city == "上海市"){ + def results = CommonDatautils.findAllNewPoiDatas(cityObj.city) + println cityObj.city+" "+results.size() + def rows = running(cityObj.city, results) + save(rows) + } + } + } + def getAllPoiType(filePath){ + def poiTypes = ExcelUtils.read(filePath, 0, 1) + def types = [] + for(def poi : poiTypes){ + types << poi['A'] + } + return types + } + + def running(city,results){ + def rets = new CopyOnWriteArrayList() + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(results.size()) + def townMap = CommonDatautils.queryFullRegionTownByCity(city, ["city","district","street", "shape"]) + for(def i=0;i=1){ + saveObj.type = arrs[0] + } + if(arrs.size()>=2){ + saveObj.secondary_type = arrs[1] + }else{ + saveObj.secondary_type = saveObj.type + } + if(arrs.size()>=3){ + def smallstr = arrs[2] + def smallArr = null + if(smallstr.contains("|")){ + smallArr = smallstr.split("|") + } + if(smallArr && smallArr.size()>=1){ + saveObj.small_type = smallArr[0] + }else{ + saveObj.small_type = smallstr + } + }else{ + saveObj.small_type = saveObj.secondary_type + } + + saveObj.cat = saveObj.type+";"+saveObj.secondary_type+";"+saveObj.small_type + + } + def location = [:] + location.lon = DataHandleUtils.getDoubleRound(6, map.lng) + location.lat = DataHandleUtils.getDoubleRound(6, map.lat) + saveObj.location = location + saveObj.spell_name = com.fangjia.core.util.DataUtil.hanyuQuanpin(map.name, " ") + saveObj.version = version + saveObj.data_id = map.map_id + saveObj.data_source = "高德地图" + saveObj.data_source_year = "2024-11" + + if(saveObj.cat && TAGS.contains(saveObj.cat)){ + saveObj.status = 0 + }else{ + saveObj.status = 1 + } + + if(location){ + def towns = townMap.get(saveObj.district) + if(towns){ + for(def town : towns){ + if(town.shape && town.shape.coordinates){ + def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates)) + if(flag){ + saveObj.township = town.street + break + } + } + } + if(!saveObj.township && map.address.contains("街道")){ + println map.address +" 含街道" + } + } + } + + if(map.address){ + if(map.address.length()<3){ + saveObj.township = "" + }else{ + def address = map.address + def i = 0 + while(i<5){ + if(address.toString().startsWith(map.province)){ + address = address.toString().substring(0, map.province.toString().length()) + } + if(address.toString().startsWith(map.city)){ + address = address.toString().substring(0, map.city.toString().length()) + } + if(address.toString().startsWith(map.district)){ + address = address.toString().substring(0, map.district.toString().length()) + } + if(address.toString().startsWith("市辖区")){ + address = address.toString().substring(0, 3) + } + if(address.toString().startsWith(saveObj.township)){ + address = address.toString().substring(0, saveObj.township.toString().length()) + } + } + saveObj.township = address + } + } + return saveObj + } + + def save(rows){ + BulkRequest request = new BulkRequest(); + int batchSize = 1000; + int processedCount = 0; + def client = ElasticUtils89.getClient("192.168.0.89") + + rows.each{key,map-> +// def map = rows.get(key) + + request.add(new IndexRequest("didata_map_poi_single", "_doc", map.id).source(map, XContentType.JSON)); + + // 批量提交数据 + if (request.numberOfActions() >= batchSize) { + request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); + if (bulkResponse.hasFailures()) { + // 处理错误 + print "Bulk request has failures: " + bulkResponse.buildFailureMessage() + } else { + processedCount += request.numberOfActions(); + println "Processed " + processedCount + " documents "+new Date().format("yyyy-MM-dd HH:mm:ss") + } + + request = new BulkRequest(); + } + } + + if (request.numberOfActions() > 0) { + request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); + if (bulkResponse.hasFailures()) { + // 处理错误 + println "Bulk request has failures: " + bulkResponse.buildFailureMessage() + } else { + processedCount += request.numberOfActions(); + println "Processed " + processedCount + " documents" + } + } + + } + +}