新版poi处理

This commit is contained in:
2024-11-25 18:34:02 +08:00
parent 1ad116d673
commit 502c682c8b
2 changed files with 263 additions and 3 deletions
@@ -4209,13 +4209,13 @@ class CommonDatautils {
static def findAllNewPoiDatas(city){
String sql = "select cityname,adname,name,address,location from poi_amap_2024 where cityname='${city}'"
def rets = [:]
String sql = "select pname,cityname,adname,name,address,location,type,id from poi_amap_2024 where cityname='${city}' limit 100"
def rets = []
def index= 0
ElasticUtils.executeSqlQuery(sql, { map->
index++
println index
rets << ["map_id":map.id,"city":map.cityname, "district":map.adname, "name":map.name, "address":map.address, "type":map.type,"lng":map.location.x, "lat":map.location.y]
rets << ["map_id":map.id,"province":map.pname,"city":map.cityname, "district":map.adname, "name":map.name, "address":map.address, "cat":map.type,"lng":map.location.x, "lat":map.location.y]
})
return rets;
}
@@ -5149,6 +5149,34 @@ class CommonDatautils {
}
static def queryFullRegionTownByCity(city, fields){
def q = new BasicDBObject()
q.put("type","乡镇街道")
q.put("city",city)
def keys = new BasicDBObject()
for(def key : fields){
keys.put(key,1)
}
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def townMap = [:]
for(def d : dbcursor){
if(d.district){
def list = townMap.get(d.district)
if(!list){
list = []
}
list << d
townMap.put(d.district, list)
}
}
println "[${city}] 乡镇街道 加载完成 "+new Date().format("yyyy-MM-dd HH:mm:ss")
return townMap
}
static def queryTownByCity(city){
@@ -0,0 +1,232 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.ExcelUtils
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.action.support.WriteRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.common.xcontent.XContentType
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 新版poi处理入库(第二批)
*/
@BatchOwner(Watcher.GaoJuan)
class NewGaoDePoiBatch extends BaseBatch{
public static def version = new Date().getTime()
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
public static def TAGS = []
static main(args) {
runBatch(new NewGaoDePoiBatch(), args)
}
@Override
void process(Object args) {
run()
}
def run(){
if (CommonDatautils.isWindow()) {
TAGS = getAllPoiType("D:\\数据\\福建省_POI_GCJ02_20240630\\POI分类白名单.xlsx")
}else{
TAGS = getAllPoiType("/home/goojia/gj/POI_TYPES.xlsx")
}
def cityList = CommonDatautils.findFullNameCitys().reverse()
for(def cityObj : cityList){
if(cityObj.city == "上海市"){
def results = CommonDatautils.findAllNewPoiDatas(cityObj.city)
println cityObj.city+" "+results.size()
def rows = running(cityObj.city, results)
save(rows)
}
}
}
def getAllPoiType(filePath){
def poiTypes = ExcelUtils.read(filePath, 0, 1)
def types = []
for(def poi : poiTypes){
types << poi['A']
}
return types
}
def running(city,results){
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(results.size())
def townMap = CommonDatautils.queryFullRegionTownByCity(city, ["city","district","street", "shape"])
for(def i=0;i<results.size();i++){
def map = results[i]
// fixedThreadPool.execute(new Runnable() {
// void run() {
try{
totalCount.addAndGet(1)
rets.addAll(createData(map, townMap))
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
// }
// });
}
// latch.await()
return rets;
}
def createData(map, townMap){
def saveObj = [:]
saveObj.province = map.province
saveObj.city = map.city
saveObj.district = map.district
saveObj.name = map.name
if(map.cat){
def arrs = map.cat.split(";|:")
if(arrs.size()>=1){
saveObj.type = arrs[0]
}
if(arrs.size()>=2){
saveObj.secondary_type = arrs[1]
}else{
saveObj.secondary_type = saveObj.type
}
if(arrs.size()>=3){
def smallstr = arrs[2]
def smallArr = null
if(smallstr.contains("|")){
smallArr = smallstr.split("|")
}
if(smallArr && smallArr.size()>=1){
saveObj.small_type = smallArr[0]
}else{
saveObj.small_type = smallstr
}
}else{
saveObj.small_type = saveObj.secondary_type
}
saveObj.cat = saveObj.type+";"+saveObj.secondary_type+";"+saveObj.small_type
}
def location = [:]
location.lon = DataHandleUtils.getDoubleRound(6, map.lng)
location.lat = DataHandleUtils.getDoubleRound(6, map.lat)
saveObj.location = location
saveObj.spell_name = com.fangjia.core.util.DataUtil.hanyuQuanpin(map.name, " ")
saveObj.version = version
saveObj.data_id = map.map_id
saveObj.data_source = "高德地图"
saveObj.data_source_year = "2024-11"
if(saveObj.cat && TAGS.contains(saveObj.cat)){
saveObj.status = 0
}else{
saveObj.status = 1
}
if(location){
def towns = townMap.get(saveObj.district)
if(towns){
for(def town : towns){
if(town.shape && town.shape.coordinates){
def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates))
if(flag){
saveObj.township = town.street
break
}
}
}
if(!saveObj.township && map.address.contains("街道")){
println map.address +" 含街道"
}
}
}
if(map.address){
if(map.address.length()<3){
saveObj.township = ""
}else{
def address = map.address
def i = 0
while(i<5){
if(address.toString().startsWith(map.province)){
address = address.toString().substring(0, map.province.toString().length())
}
if(address.toString().startsWith(map.city)){
address = address.toString().substring(0, map.city.toString().length())
}
if(address.toString().startsWith(map.district)){
address = address.toString().substring(0, map.district.toString().length())
}
if(address.toString().startsWith("市辖区")){
address = address.toString().substring(0, 3)
}
if(address.toString().startsWith(saveObj.township)){
address = address.toString().substring(0, saveObj.township.toString().length())
}
}
saveObj.township = address
}
}
return saveObj
}
def save(rows){
BulkRequest request = new BulkRequest();
int batchSize = 1000;
int processedCount = 0;
def client = ElasticUtils89.getClient("192.168.0.89")
rows.each{key,map->
// def map = rows.get(key)
request.add(new IndexRequest("didata_map_poi_single", "_doc", map.id).source(map, XContentType.JSON));
// 批量提交数据
if (request.numberOfActions() >= batchSize) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
print "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents "+new Date().format("yyyy-MM-dd HH:mm:ss")
}
request = new BulkRequest();
}
}
if (request.numberOfActions() > 0) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents"
}
}
}
}