poi入库优化
This commit is contained in:
+38
-15
@@ -128,7 +128,7 @@ class GaoDePoiDataProcess extends BaseBatch{
|
|||||||
}
|
}
|
||||||
|
|
||||||
def sysnchVillage(){
|
def sysnchVillage(){
|
||||||
def buducitys = [ "湖州市","舟山市","宁波市","丽水市","杭州市","温州市","嘉兴市","台州市","绍兴市","金华市","衢州市",'茂名市',
|
def buducitys = [ "宁波市","丽水市","杭州市","温州市","嘉兴市","台州市","绍兴市","金华市","衢州市",'茂名市',
|
||||||
'荆州市', '荆门市', '莆田市','苏州市','菏泽市', '萍乡市', '营口市', '葫芦岛市', '蚌埠市', '衡水市', '衡阳市',
|
'荆州市', '荆门市', '莆田市','苏州市','菏泽市', '萍乡市', '营口市', '葫芦岛市', '蚌埠市', '衡水市', '衡阳市',
|
||||||
'衢州市', '襄阳市', '西宁市', '西安市', '许昌市', '贵港市', '贵阳市', '贺州市', '资阳市', '赣州市', '赤峰市',
|
'衢州市', '襄阳市', '西宁市', '西安市', '许昌市', '贵港市', '贵阳市', '贺州市', '资阳市', '赣州市', '赤峰市',
|
||||||
'辽源市', '辽阳市', '达州市', '运城市', '连云港市', '通化市', '通辽市', '遂宁市', '遵义市', '中山市', '临夏回族自治州',
|
'辽源市', '辽阳市', '达州市', '运城市', '连云港市', '通化市', '通辽市', '遂宁市', '遵义市', '中山市', '临夏回族自治州',
|
||||||
@@ -168,7 +168,7 @@ class GaoDePoiDataProcess extends BaseBatch{
|
|||||||
'鸡西市', '鹤壁市', '鹤岗市', '鹰潭市', '黄冈市', '黄山市', '黄石市', '黑河市',
|
'鸡西市', '鹤壁市', '鹤岗市', '鹰潭市', '黄冈市', '黄山市', '黄石市', '黑河市',
|
||||||
'黔东南苗族侗族自治州', '黔南布依族苗族自治州', '黔西南布依族苗族自治州', '齐齐哈尔市', '龙岩市']
|
'黔东南苗族侗族自治州', '黔南布依族苗族自治州', '黔西南布依族苗族自治州', '齐齐哈尔市', '龙岩市']
|
||||||
|
|
||||||
def runbuducitys = []
|
def runbuducitys = ["舟山市","湖州市"]
|
||||||
for(def city : buducitys){
|
for(def city : buducitys){
|
||||||
if(runbuducitys.contains(city)){
|
if(runbuducitys.contains(city)){
|
||||||
continue
|
continue
|
||||||
@@ -205,7 +205,10 @@ class GaoDePoiDataProcess extends BaseBatch{
|
|||||||
def prov = ProvinceData.getProvince(p.city.replace("市",""))
|
def prov = ProvinceData.getProvince(p.city.replace("市",""))
|
||||||
def ret = createVillageData(prov, p, townMap)
|
def ret = createVillageData(prov, p, townMap)
|
||||||
if(ret){
|
if(ret){
|
||||||
def simlarObj = findSimlarPoi(ret, esMap)
|
def simlarObj = null
|
||||||
|
if(esMap){
|
||||||
|
simlarObj = findSimlarPoi(ret, esMap)
|
||||||
|
}
|
||||||
if(!simlarObj){
|
if(!simlarObj){
|
||||||
def newKey = ret.city+"##"+ret.district+"##"+ret.name.replace("-","").replace("·","")
|
def newKey = ret.city+"##"+ret.district+"##"+ret.name.replace("-","").replace("·","")
|
||||||
if(!rets.containsKey(newKey)){
|
if(!rets.containsKey(newKey)){
|
||||||
@@ -227,19 +230,26 @@ class GaoDePoiDataProcess extends BaseBatch{
|
|||||||
}
|
}
|
||||||
|
|
||||||
def findSimlarPoi(ret, esMap){
|
def findSimlarPoi(ret, esMap){
|
||||||
def keys = esMap.keySet()
|
|
||||||
def simlarObj = null
|
def simlarObj = null
|
||||||
for(def key : keys){
|
def region = ""
|
||||||
def esValue = esMap.get(key)
|
if(ret.district){
|
||||||
if(esValue.township && ret.township){
|
region = ret.district
|
||||||
if(esValue.name.contains(ret.name) || ret.name.contains(esValue.name)){//||
|
}
|
||||||
def distance = GeoUtils.calculateDistance(ret.location.lon, ret.location.lat, esValue.location.lon, esValue.location.lat);
|
def regionMap = esMap.get(region)
|
||||||
if(distance <100){
|
if(regionMap){
|
||||||
simlarObj = esValue
|
def keys = regionMap.keySet()
|
||||||
println esValue.name +" "+ret.name+" "+distance
|
|
||||||
break;
|
for(def key : keys){
|
||||||
|
def esValue = regionMap.get(key)
|
||||||
|
if(esValue.township && ret.township){
|
||||||
|
if(esValue.name.contains(ret.name) || ret.name.contains(esValue.name)){//||
|
||||||
|
def distance = GeoUtils.calculateDistance(ret.location.lon, ret.location.lat, esValue.location.lon, esValue.location.lat);
|
||||||
|
if(distance <100){
|
||||||
|
simlarObj = esValue
|
||||||
|
println esValue.name +" "+ret.name+" "+distance
|
||||||
|
break;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
// else{
|
// else{
|
||||||
// def esKey = esValue.name.replace("-","").replace("·","").replace("(", "").replace(")", "")
|
// def esKey = esValue.name.replace("-","").replace("·","").replace("(", "").replace(")", "")
|
||||||
// def bdKey = ret.name.replace("-","").replace("·","").replace("(", "").replace(")", "")
|
// def bdKey = ret.name.replace("-","").replace("·","").replace("(", "").replace(")", "")
|
||||||
@@ -253,8 +263,12 @@ class GaoDePoiDataProcess extends BaseBatch{
|
|||||||
// }
|
// }
|
||||||
// }
|
// }
|
||||||
// }
|
// }
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
}else{
|
||||||
|
println "无数据"
|
||||||
}
|
}
|
||||||
|
|
||||||
return simlarObj
|
return simlarObj
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -264,13 +278,22 @@ class GaoDePoiDataProcess extends BaseBatch{
|
|||||||
// BloomFilter<CharSequence> filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001);
|
// BloomFilter<CharSequence> filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001);
|
||||||
def map = new HashMap()
|
def map = new HashMap()
|
||||||
ElasticUtils89.executeQuery("192.168.0.89", "didata_map_poi_single", queryBuilder, { String id, Map d->
|
ElasticUtils89.executeQuery("192.168.0.89", "didata_map_poi_single", queryBuilder, { String id, Map d->
|
||||||
|
def district = ""
|
||||||
|
if(d.district){
|
||||||
|
district = d.district
|
||||||
|
}
|
||||||
|
def regionMap = map.get(district)
|
||||||
|
if(!regionMap){
|
||||||
|
regionMap = new HashMap()
|
||||||
|
}
|
||||||
def o = new BasicDBObject()
|
def o = new BasicDBObject()
|
||||||
o.put("city", d.city)
|
o.put("city", d.city)
|
||||||
o.put("district", d.district)
|
o.put("district", d.district)
|
||||||
o.put("name", d.name)
|
o.put("name", d.name)
|
||||||
o.put("township", d.township)
|
o.put("township", d.township)
|
||||||
o.put("location", d.location)
|
o.put("location", d.location)
|
||||||
map.put(d.city +"##"+d.district+"##"+d.name.replace("-","").replace("·",""), o)
|
regionMap.put(d.name.replace("-","").replace("·",""), o)
|
||||||
|
map.put(district, regionMap)
|
||||||
})
|
})
|
||||||
return map
|
return map
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user