poi入库优化

This commit is contained in:
2024-08-28 17:34:36 +08:00
parent bfca3691e5
commit 51138cb628
@@ -128,7 +128,7 @@ class GaoDePoiDataProcess extends BaseBatch{
}
def sysnchVillage(){
def buducitys = [ "湖州市","舟山市","宁波市","丽水市","杭州市","温州市","嘉兴市","台州市","绍兴市","金华市","衢州市",'茂名市',
def buducitys = [ "宁波市","丽水市","杭州市","温州市","嘉兴市","台州市","绍兴市","金华市","衢州市",'茂名市',
'荆州市', '荆门市', '莆田市','苏州市','菏泽市', '萍乡市', '营口市', '葫芦岛市', '蚌埠市', '衡水市', '衡阳市',
'衢州市', '襄阳市', '西宁市', '西安市', '许昌市', '贵港市', '贵阳市', '贺州市', '资阳市', '赣州市', '赤峰市',
'辽源市', '辽阳市', '达州市', '运城市', '连云港市', '通化市', '通辽市', '遂宁市', '遵义市', '中山市', '临夏回族自治州',
@@ -168,7 +168,7 @@ class GaoDePoiDataProcess extends BaseBatch{
'鸡西市', '鹤壁市', '鹤岗市', '鹰潭市', '黄冈市', '黄山市', '黄石市', '黑河市',
'黔东南苗族侗族自治州', '黔南布依族苗族自治州', '黔西南布依族苗族自治州', '齐齐哈尔市', '龙岩市']
def runbuducitys = []
def runbuducitys = ["舟山市","湖州市"]
for(def city : buducitys){
if(runbuducitys.contains(city)){
continue
@@ -205,7 +205,10 @@ class GaoDePoiDataProcess extends BaseBatch{
def prov = ProvinceData.getProvince(p.city.replace("市",""))
def ret = createVillageData(prov, p, townMap)
if(ret){
def simlarObj = findSimlarPoi(ret, esMap)
def simlarObj = null
if(esMap){
simlarObj = findSimlarPoi(ret, esMap)
}
if(!simlarObj){
def newKey = ret.city+"##"+ret.district+"##"+ret.name.replace("-","").replace("·","")
if(!rets.containsKey(newKey)){
@@ -227,19 +230,26 @@ class GaoDePoiDataProcess extends BaseBatch{
}
def findSimlarPoi(ret, esMap){
def keys = esMap.keySet()
def simlarObj = null
for(def key : keys){
def esValue = esMap.get(key)
if(esValue.township && ret.township){
if(esValue.name.contains(ret.name) || ret.name.contains(esValue.name)){//||
def distance = GeoUtils.calculateDistance(ret.location.lon, ret.location.lat, esValue.location.lon, esValue.location.lat);
if(distance <100){
simlarObj = esValue
println esValue.name +" "+ret.name+" "+distance
break;
def region = ""
if(ret.district){
region = ret.district
}
def regionMap = esMap.get(region)
if(regionMap){
def keys = regionMap.keySet()
for(def key : keys){
def esValue = regionMap.get(key)
if(esValue.township && ret.township){
if(esValue.name.contains(ret.name) || ret.name.contains(esValue.name)){//||
def distance = GeoUtils.calculateDistance(ret.location.lon, ret.location.lat, esValue.location.lon, esValue.location.lat);
if(distance <100){
simlarObj = esValue
println esValue.name +" "+ret.name+" "+distance
break;
}
}
}
// else{
// def esKey = esValue.name.replace("-","").replace("·","").replace("(", "").replace(")", "")
// def bdKey = ret.name.replace("-","").replace("·","").replace("(", "").replace(")", "")
@@ -253,8 +263,12 @@ class GaoDePoiDataProcess extends BaseBatch{
// }
// }
// }
}
}
}else{
println "无数据"
}
return simlarObj
}
@@ -264,13 +278,22 @@ class GaoDePoiDataProcess extends BaseBatch{
// BloomFilter<CharSequence> filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001);
def map = new HashMap()
ElasticUtils89.executeQuery("192.168.0.89", "didata_map_poi_single", queryBuilder, { String id, Map d->
def district = ""
if(d.district){
district = d.district
}
def regionMap = map.get(district)
if(!regionMap){
regionMap = new HashMap()
}
def o = new BasicDBObject()
o.put("city", d.city)
o.put("district", d.district)
o.put("name", d.name)
o.put("township", d.township)
o.put("location", d.location)
map.put(d.city +"##"+d.district+"##"+d.name.replace("-","").replace("·",""), o)
regionMap.put(d.name.replace("-","").replace("·",""), o)
map.put(district, regionMap)
})
return map
}