From 046be14df5116841feef1dffaba5960bc85b76b3 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Thu, 22 Aug 2024 18:09:53 +0800 Subject: [PATCH] update --- .../gj/xiamen/GaoDePoiDataProcess.groovy | 24 +++++++------------ 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy index 0a4f07a1c..173c8a5dc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy @@ -126,7 +126,9 @@ class GaoDePoiDataProcess extends BaseBatch{ } def sysnchVillage(){ - def buducitys = [ '临汾市', '临沂市', '临沧市', '临高县', '丹东市', + def buducitys = [ "湖州市","宁波市","丽水市","杭州市","温州市","嘉兴市","台州市","绍兴市","金华市","舟山市","衢州市", + '绍兴市','七台河市','万宁市', '三亚市', '三明市', '三门峡市', '上饶市', '东方市', '东莞市','上海市', + '东营市', '中卫市', '中山市', '临夏回族自治州', '临汾市', '临沂市', '临沧市', '临高县', '丹东市', '丽水市', '丽江市', '乌兰察布市', '乌海市', '乌鲁木齐市', '乐东黎族自治县', '乐山市', '九江市', '云浮市', '五家渠市', '五指山市', '亳州市', '仙桃市', '伊春市', '伊犁哈萨克自治州', '佛山市', '佳木斯市', '保亭黎族苗族自治县', '保定市', '保山市', '信阳市', '儋州市', '克孜勒苏柯尔克孜自治州', @@ -154,7 +156,7 @@ class GaoDePoiDataProcess extends BaseBatch{ '潜江市', '潮州市', '澄迈县', '濮阳市', '烟台市', '焦作市', '牡丹江市', '玉林市', '玉溪市', '珠海市', '琼中黎族苗族自治县', '琼海市', '甘南藏族自治州', '白城市', '白山市', '白沙黎族自治县', '白银市', '百色市', '益阳市', '盐城市', '盘锦市', '眉山市', '石嘴山市', '石家庄市', '石河子市', - '神农架林区', '福州市', '秦皇岛市', '红河哈尼族彝族自治州', '绍兴市', '绥化市', '绵阳市', '聊城市', + '神农架林区', '福州市', '秦皇岛市', '红河哈尼族彝族自治州', '绥化市', '绵阳市', '聊城市', '肇庆市', '自贡市', '舟山市', '芜湖市', '苏州市', '茂名市', '荆州市', '荆门市', '莆田市', '菏泽市', '萍乡市', '营口市', '葫芦岛市', '蚌埠市', '衡水市', '衡阳市', '衢州市', '襄阳市', '西宁市', '西安市', '许昌市', '贵港市', '贵阳市', '贺州市', '资阳市', '赣州市', '赤峰市', @@ -167,9 +169,7 @@ class GaoDePoiDataProcess extends BaseBatch{ '鸡西市', '鹤壁市', '鹤岗市', '鹰潭市', '黄冈市', '黄山市', '黄石市', '黑河市', '黔东南苗族侗族自治州', '黔南布依族苗族自治州', '黔西南布依族苗族自治州', '齐齐哈尔市', '龙岩市'] - //"丽水市","杭州市", - def runbuducitys = ["湖州市","宁波市","温州市","嘉兴市","台州市","绍兴市","金华市","舟山市","衢州市",'七台河市', '上海市','万宁市', '三亚市', '三明市', '三门峡市', '上饶市', '东方市', '东莞市', - '东营市', '中卫市', '中山市', '临夏回族自治州'] + def runbuducitys = [] for(def city : buducitys){ if(runbuducitys.contains(city)){ continue @@ -177,7 +177,6 @@ class GaoDePoiDataProcess extends BaseBatch{ def pois = findAllVillagePoiMap(city) def rows = createVillageData(city, pois) - save(city, rows) // saveToMG(city, rows) println city +" "+pois.size()+" add:"+rows.size() @@ -206,7 +205,7 @@ class GaoDePoiDataProcess extends BaseBatch{ def ret = createVillageData(prov, p, townMap) if(!keyRets.contains(ret.city+"##"+ret.district+"##"+ret.name)){ - keyRets.addAll(ret) + keyRets << ret.city+"##"+ret.district+"##"+ret.name rets << ret } } @@ -225,16 +224,9 @@ class GaoDePoiDataProcess extends BaseBatch{ def findEsPoiByCity(city){ -// def sql = "select city,district,name,location from didata_map_poi_single where city='${city}' " -// def rets = new HashSet() -// ElasticUtils89.executeSqlQuery(sql, { d-> -// -// }) QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)) -// def rets = new HashSet() BloomFilter filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001); ElasticUtils89.executeQuery("192.168.0.89", "didata_map_poi_single", queryBuilder, { String id, Map d-> -// rets << d.city +"##"+d.district+"##"+d.name filter.put(d.city +"##"+d.district+"##"+d.name) }) return filter @@ -429,7 +421,7 @@ class GaoDePoiDataProcess extends BaseBatch{ keys.put("gcj02_x", 1) keys.put("gcj02_y", 1) keys.put("std_tag", 1) - def list = BAIDU_POI_COLL.find(q, keys).toList() + def list = BAIDU_POI_COLL.find(q, keys).batchSize(10000).toList() return list; } @@ -717,7 +709,7 @@ class GaoDePoiDataProcess extends BaseBatch{ print "Bulk request has failures: " + bulkResponse.buildFailureMessage() } else { processedCount += request.numberOfActions(); - println "Processed " + processedCount + " documents" + println "Processed " + processedCount + " documents "+new Date().format("yyyy-MM-dd HH:mm:ss") } request = new BulkRequest();