Merge branch 'script_gj' into 'master'

update

See merge request !921
This commit is contained in:
2024-08-22 18:10:59 +08:00
@@ -126,7 +126,9 @@ class GaoDePoiDataProcess extends BaseBatch{
}
def sysnchVillage(){
def buducitys = [ '临汾市', '临沂市', '临沧市', '临高县', '丹东市',
def buducitys = [ "湖州市","宁波市","丽水市","杭州市","温州市","嘉兴市","台州市","绍兴市","金华市","舟山市","衢州市",
'绍兴市','七台河市','万宁市', '三亚市', '三明市', '三门峡市', '上饶市', '东方市', '东莞市','上海市',
'东营市', '中卫市', '中山市', '临夏回族自治州', '临汾市', '临沂市', '临沧市', '临高县', '丹东市',
'丽水市', '丽江市', '乌兰察布市', '乌海市', '乌鲁木齐市', '乐东黎族自治县', '乐山市', '九江市',
'云浮市', '五家渠市', '五指山市', '亳州市', '仙桃市', '伊春市', '伊犁哈萨克自治州', '佛山市',
'佳木斯市', '保亭黎族苗族自治县', '保定市', '保山市', '信阳市', '儋州市', '克孜勒苏柯尔克孜自治州',
@@ -154,7 +156,7 @@ class GaoDePoiDataProcess extends BaseBatch{
'潜江市', '潮州市', '澄迈县', '濮阳市', '烟台市', '焦作市', '牡丹江市', '玉林市', '玉溪市',
'珠海市', '琼中黎族苗族自治县', '琼海市', '甘南藏族自治州', '白城市', '白山市', '白沙黎族自治县',
'白银市', '百色市', '益阳市', '盐城市', '盘锦市', '眉山市', '石嘴山市', '石家庄市', '石河子市',
'神农架林区', '福州市', '秦皇岛市', '红河哈尼族彝族自治州', '绍兴市', '绥化市', '绵阳市', '聊城市',
'神农架林区', '福州市', '秦皇岛市', '红河哈尼族彝族自治州', '绥化市', '绵阳市', '聊城市',
'肇庆市', '自贡市', '舟山市', '芜湖市', '苏州市', '茂名市', '荆州市', '荆门市', '莆田市',
'菏泽市', '萍乡市', '营口市', '葫芦岛市', '蚌埠市', '衡水市', '衡阳市', '衢州市', '襄阳市',
'西宁市', '西安市', '许昌市', '贵港市', '贵阳市', '贺州市', '资阳市', '赣州市', '赤峰市',
@@ -167,9 +169,7 @@ class GaoDePoiDataProcess extends BaseBatch{
'鸡西市', '鹤壁市', '鹤岗市', '鹰潭市', '黄冈市', '黄山市', '黄石市', '黑河市',
'黔东南苗族侗族自治州', '黔南布依族苗族自治州', '黔西南布依族苗族自治州', '齐齐哈尔市', '龙岩市']
//"丽水市","杭州市",
def runbuducitys = ["湖州市","宁波市","温州市","嘉兴市","台州市","绍兴市","金华市","舟山市","衢州市",'七台河市', '上海市','万宁市', '三亚市', '三明市', '三门峡市', '上饶市', '东方市', '东莞市',
'东营市', '中卫市', '中山市', '临夏回族自治州']
def runbuducitys = []
for(def city : buducitys){
if(runbuducitys.contains(city)){
continue
@@ -177,7 +177,6 @@ class GaoDePoiDataProcess extends BaseBatch{
def pois = findAllVillagePoiMap(city)
def rows = createVillageData(city, pois)
save(city, rows)
// saveToMG(city, rows)
println city +" "+pois.size()+" add:"+rows.size()
@@ -206,7 +205,7 @@ class GaoDePoiDataProcess extends BaseBatch{
def ret = createVillageData(prov, p, townMap)
if(!keyRets.contains(ret.city+"##"+ret.district+"##"+ret.name)){
keyRets.addAll(ret)
keyRets << ret.city+"##"+ret.district+"##"+ret.name
rets << ret
}
}
@@ -225,16 +224,9 @@ class GaoDePoiDataProcess extends BaseBatch{
def findEsPoiByCity(city){
// def sql = "select city,district,name,location from didata_map_poi_single where city='${city}' "
// def rets = new HashSet()
// ElasticUtils89.executeSqlQuery(sql, { d->
//
// })
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
// def rets = new HashSet()
BloomFilter<CharSequence> filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001);
ElasticUtils89.executeQuery("192.168.0.89", "didata_map_poi_single", queryBuilder, { String id, Map d->
// rets << d.city +"##"+d.district+"##"+d.name
filter.put(d.city +"##"+d.district+"##"+d.name)
})
return filter
@@ -429,7 +421,7 @@ class GaoDePoiDataProcess extends BaseBatch{
keys.put("gcj02_x", 1)
keys.put("gcj02_y", 1)
keys.put("std_tag", 1)
def list = BAIDU_POI_COLL.find(q, keys).toList()
def list = BAIDU_POI_COLL.find(q, keys).batchSize(10000).toList()
return list;
}
@@ -717,7 +709,7 @@ class GaoDePoiDataProcess extends BaseBatch{
print "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents"
println "Processed " + processedCount + " documents "+new Date().format("yyyy-MM-dd HH:mm:ss")
}
request = new BulkRequest();