From 046be14df5116841feef1dffaba5960bc85b76b3 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Thu, 22 Aug 2024 18:09:53 +0800 Subject: [PATCH 1/6] update --- .../gj/xiamen/GaoDePoiDataProcess.groovy | 24 +++++++------------ 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy index 0a4f07a1c..173c8a5dc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy @@ -126,7 +126,9 @@ class GaoDePoiDataProcess extends BaseBatch{ } def sysnchVillage(){ - def buducitys = [ '临汾市', '临沂市', '临沧市', '临高县', '丹东市', + def buducitys = [ "湖州市","宁波市","丽水市","杭州市","温州市","嘉兴市","台州市","绍兴市","金华市","舟山市","衢州市", + '绍兴市','七台河市','万宁市', '三亚市', '三明市', '三门峡市', '上饶市', '东方市', '东莞市','上海市', + '东营市', '中卫市', '中山市', '临夏回族自治州', '临汾市', '临沂市', '临沧市', '临高县', '丹东市', '丽水市', '丽江市', '乌兰察布市', '乌海市', '乌鲁木齐市', '乐东黎族自治县', '乐山市', '九江市', '云浮市', '五家渠市', '五指山市', '亳州市', '仙桃市', '伊春市', '伊犁哈萨克自治州', '佛山市', '佳木斯市', '保亭黎族苗族自治县', '保定市', '保山市', '信阳市', '儋州市', '克孜勒苏柯尔克孜自治州', @@ -154,7 +156,7 @@ class GaoDePoiDataProcess extends BaseBatch{ '潜江市', '潮州市', '澄迈县', '濮阳市', '烟台市', '焦作市', '牡丹江市', '玉林市', '玉溪市', '珠海市', '琼中黎族苗族自治县', '琼海市', '甘南藏族自治州', '白城市', '白山市', '白沙黎族自治县', '白银市', '百色市', '益阳市', '盐城市', '盘锦市', '眉山市', '石嘴山市', '石家庄市', '石河子市', - '神农架林区', '福州市', '秦皇岛市', '红河哈尼族彝族自治州', '绍兴市', '绥化市', '绵阳市', '聊城市', + '神农架林区', '福州市', '秦皇岛市', '红河哈尼族彝族自治州', '绥化市', '绵阳市', '聊城市', '肇庆市', '自贡市', '舟山市', '芜湖市', '苏州市', '茂名市', '荆州市', '荆门市', '莆田市', '菏泽市', '萍乡市', '营口市', '葫芦岛市', '蚌埠市', '衡水市', '衡阳市', '衢州市', '襄阳市', '西宁市', '西安市', '许昌市', '贵港市', '贵阳市', '贺州市', '资阳市', '赣州市', '赤峰市', @@ -167,9 +169,7 @@ class GaoDePoiDataProcess extends BaseBatch{ '鸡西市', '鹤壁市', '鹤岗市', '鹰潭市', '黄冈市', '黄山市', '黄石市', '黑河市', '黔东南苗族侗族自治州', '黔南布依族苗族自治州', '黔西南布依族苗族自治州', '齐齐哈尔市', '龙岩市'] - //"丽水市","杭州市", - def runbuducitys = ["湖州市","宁波市","温州市","嘉兴市","台州市","绍兴市","金华市","舟山市","衢州市",'七台河市', '上海市','万宁市', '三亚市', '三明市', '三门峡市', '上饶市', '东方市', '东莞市', - '东营市', '中卫市', '中山市', '临夏回族自治州'] + def runbuducitys = [] for(def city : buducitys){ if(runbuducitys.contains(city)){ continue @@ -177,7 +177,6 @@ class GaoDePoiDataProcess extends BaseBatch{ def pois = findAllVillagePoiMap(city) def rows = createVillageData(city, pois) - save(city, rows) // saveToMG(city, rows) println city +" "+pois.size()+" add:"+rows.size() @@ -206,7 +205,7 @@ class GaoDePoiDataProcess extends BaseBatch{ def ret = createVillageData(prov, p, townMap) if(!keyRets.contains(ret.city+"##"+ret.district+"##"+ret.name)){ - keyRets.addAll(ret) + keyRets << ret.city+"##"+ret.district+"##"+ret.name rets << ret } } @@ -225,16 +224,9 @@ class GaoDePoiDataProcess extends BaseBatch{ def findEsPoiByCity(city){ -// def sql = "select city,district,name,location from didata_map_poi_single where city='${city}' " -// def rets = new HashSet() -// ElasticUtils89.executeSqlQuery(sql, { d-> -// -// }) QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)) -// def rets = new HashSet() BloomFilter filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001); ElasticUtils89.executeQuery("192.168.0.89", "didata_map_poi_single", queryBuilder, { String id, Map d-> -// rets << d.city +"##"+d.district+"##"+d.name filter.put(d.city +"##"+d.district+"##"+d.name) }) return filter @@ -429,7 +421,7 @@ class GaoDePoiDataProcess extends BaseBatch{ keys.put("gcj02_x", 1) keys.put("gcj02_y", 1) keys.put("std_tag", 1) - def list = BAIDU_POI_COLL.find(q, keys).toList() + def list = BAIDU_POI_COLL.find(q, keys).batchSize(10000).toList() return list; } @@ -717,7 +709,7 @@ class GaoDePoiDataProcess extends BaseBatch{ print "Bulk request has failures: " + bulkResponse.buildFailureMessage() } else { processedCount += request.numberOfActions(); - println "Processed " + processedCount + " documents" + println "Processed " + processedCount + " documents "+new Date().format("yyyy-MM-dd HH:mm:ss") } request = new BulkRequest(); From 8370e453817f04dd9c40c45c3d06041849163d31 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Thu, 22 Aug 2024 18:50:47 +0800 Subject: [PATCH 2/6] =?UTF-8?q?=E5=8A=A0=E5=85=A5=E6=97=B6=E9=97=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy index 173c8a5dc..669d093e2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy @@ -175,6 +175,7 @@ class GaoDePoiDataProcess extends BaseBatch{ continue } def pois = findAllVillagePoiMap(city) + println "百度loading finish: "+pois.size()+" "+new Date().format("yyyy-MM-dd HH:mm:ss") def rows = createVillageData(city, pois) save(city, rows) @@ -192,6 +193,7 @@ class GaoDePoiDataProcess extends BaseBatch{ // def townMap = findTownRange(city) def townMap = CommonDatautils.queryTownByCity(city) def poiKeysFilter = findEsPoiByCity(city) + println "Es 加载完成:"+new Date().format("yyyy-MM-dd HH:mm:ss") for(def i=0;i Date: Thu, 22 Aug 2024 19:01:18 +0800 Subject: [PATCH 3/6] =?UTF-8?q?10000-=E3=80=8B2000?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy index 669d093e2..74f9a96f5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy @@ -423,7 +423,7 @@ class GaoDePoiDataProcess extends BaseBatch{ keys.put("gcj02_x", 1) keys.put("gcj02_y", 1) keys.put("std_tag", 1) - def list = BAIDU_POI_COLL.find(q, keys).batchSize(10000).toList() + def list = BAIDU_POI_COLL.find(q, keys).batchSize(2000).toList() return list; } From 6406f2d94aaa34343befb2067a38cd2a8e79c9b2 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Thu, 22 Aug 2024 19:26:07 +0800 Subject: [PATCH 4/6] =?UTF-8?q?=E5=9C=B0=E5=9D=80=E6=8B=86=E5=88=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/gj/data_handle/MinglujiData.groovy | 41 +++++++++++++++---- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy index 538bc0a44..7032e3d85 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy @@ -31,15 +31,15 @@ class MinglujiData { def run(){ def prov = "浙江省" - def city = "嘉兴" + def city = "湖州" // exportDatas(city) // matchDev(prov, city) - matchCommunity(city) +// matchCommunity(city) // tempMatchDev(prov) - - // saveMljData() + + fiveLevelSplit(city) } def exportDatas(city){ def datas = findMinglujiDatas(city) @@ -48,11 +48,11 @@ class MinglujiData { def matchDev(prov, city){ def devs = loadAllDevs(prov) - def i = 1; + def i = 2; while(i<20) { def fileName = "/home/goojia/gj/mlj_${i}.xlsx" if (CommonDatautils.isWindow()) { - fileName = "D:\\数据\\数据建设\\名录集\\${city}_${i}.xlsx" + fileName = "D:\\数据\\数据建设\\名录集\\mlj_${i}.xlsx" } File file=new File(fileName); if(file.exists()){ @@ -152,11 +152,11 @@ class MinglujiData { } def matchCommunity(city){ - def i = 1; + def i = 8; while(i<20){ def fileName = "/home/goojia/gj/mlj_${i}.xlsx" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\数据建设\\名录集\\${city}_${i}.xlsx" + fileName = "D:\\数据\\数据建设\\名录集\\mlj_${i}.xlsx" } File file=new File(fileName); @@ -203,6 +203,29 @@ class MinglujiData { } + def fiveLevelSplit(city){ + def i = 1; + while(i<20) { + def fileName = "/home/goojia/gj/mlj_${i}.xlsx" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\数据建设\\名录集\\${city}_${i}.xlsx" + } + + File file = new File(fileName); + if (file.exists()) { + ExcelUtils.multiThreadingModify(fileName, { d -> + def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatFivelevel(d['B'], d['E']) + if(jsonObject) { + if (jsonObject.get("address") && jsonObject.get("address") != "null") { + d['J'] = jsonObject.get("address") + } + } + }) + } + i++ + } + } + def saveMljData(){ ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\名录集\\湖州入库数据.xlsx",{d-> def obj = new BasicDBObject() @@ -213,7 +236,7 @@ class MinglujiData { obj.put("address", d['E']) obj.put("dev_name", d['F']) obj.put("committee", d['G']) - obj.put("street", d['H']) + obj.put("township", d['H']) company_100_CONN.save(obj) }) } From b749597573a0ee58c2a97890c6e626c14ce6327a Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Fri, 23 Aug 2024 09:56:20 +0800 Subject: [PATCH 5/6] =?UTF-8?q?=E5=8E=BB=E9=87=8D=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gj/xiamen/GaoDePoiDataProcess.groovy | 26 +++++++++---------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy index 74f9a96f5..eacf43517 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy @@ -126,9 +126,10 @@ class GaoDePoiDataProcess extends BaseBatch{ } def sysnchVillage(){ - def buducitys = [ "湖州市","宁波市","丽水市","杭州市","温州市","嘉兴市","台州市","绍兴市","金华市","舟山市","衢州市", - '绍兴市','七台河市','万宁市', '三亚市', '三明市', '三门峡市', '上饶市', '东方市', '东莞市','上海市', - '东营市', '中卫市', '中山市', '临夏回族自治州', '临汾市', '临沂市', '临沧市', '临高县', '丹东市', + def buducitys = [ '绵阳市', '聊城市', '肇庆市', '自贡市', '芜湖市', '苏州市', '茂名市', '荆州市', '荆门市', '莆田市', + '菏泽市', '萍乡市', '营口市', '葫芦岛市', '蚌埠市', '衡水市', '衡阳市', '衢州市', '襄阳市', + '西宁市', '西安市', '许昌市', '贵港市', '贵阳市', '贺州市', '资阳市', '赣州市', '赤峰市', + '辽源市', '辽阳市', '达州市', '运城市', '连云港市', '通化市', '通辽市', '遂宁市', '遵义市', '中山市', '临夏回族自治州', '临汾市', '临沂市', '临沧市', '临高县', '丹东市', '丽水市', '丽江市', '乌兰察布市', '乌海市', '乌鲁木齐市', '乐东黎族自治县', '乐山市', '九江市', '云浮市', '五家渠市', '五指山市', '亳州市', '仙桃市', '伊春市', '伊犁哈萨克自治州', '佛山市', '佳木斯市', '保亭黎族苗族自治县', '保定市', '保山市', '信阳市', '儋州市', '克孜勒苏柯尔克孜自治州', @@ -138,7 +139,7 @@ class GaoDePoiDataProcess extends BaseBatch{ '吕梁市', '吴忠市', '周口市', '呼伦贝尔市', '呼和浩特市', '和田地区', '咸宁市', '咸阳市', '哈密地区', '哈尔滨市', '唐山市', '商丘市', '商洛市', '喀什地区', '嘉兴市', '嘉峪关市', '四平市', '固原市', '图木舒克市', '塔城地区', '大兴安岭地区', '大同市', '大庆市', '大理白族自治州', '大连市', - '天水市', '天津市', '天门市', '太原市', '威海市', '娄底市', '孝感市', '宁德市', '宁波市', + '天水市', '天津市', '天门市', '太原市', '威海市', '娄底市', '孝感市', '宁德市', '安庆市', '安康市', '安阳市', '安顺市', '定安县', '定西市', '宜宾市', '宜昌市', '宜春市', '宝鸡市', '宣城市', '宿州市', '宿迁市', '屯昌县', '山南地区', '岳阳市', '崇左市', '巴中市', '巴彦淖尔市', '巴音郭楞蒙古自治州', '常州市', '常德市', '平凉市', '平顶山市', '广元市', '广安市', @@ -151,17 +152,12 @@ class GaoDePoiDataProcess extends BaseBatch{ '梅州市', '梧州市', '楚雄彝族自治州', '榆林市', '武威市', '武汉市', '毕节地区', '永州市', '汉中市', '汕头市', '汕尾市', '江门市', '池州市', '沈阳市', '沧州市', '河池市', '河源市', '泉州市', '泰安市', '泰州市', '泸州市', '洛阳市', '济南市', '济宁市', '济源市', '海东地区', '海口市', - '淄博市', '淮北市', '淮南市', '淮安市', '深圳市', '清远市', '温州市', '渭南市', '湖州市', + '淄博市', '淮北市', '淮南市', '淮安市', '深圳市', '清远市', '温州市', '渭南市', '湘潭市', '湘西土家族苗族自治州', '湛江市', '滁州市', '滨州市', '漯河市', '漳州市', '潍坊市', '潜江市', '潮州市', '澄迈县', '濮阳市', '烟台市', '焦作市', '牡丹江市', '玉林市', '玉溪市', '珠海市', '琼中黎族苗族自治县', '琼海市', '甘南藏族自治州', '白城市', '白山市', '白沙黎族自治县', '白银市', '百色市', '益阳市', '盐城市', '盘锦市', '眉山市', '石嘴山市', '石家庄市', '石河子市', - '神农架林区', '福州市', '秦皇岛市', '红河哈尼族彝族自治州', '绥化市', '绵阳市', '聊城市', - '肇庆市', '自贡市', '舟山市', '芜湖市', '苏州市', '茂名市', '荆州市', '荆门市', '莆田市', - '菏泽市', '萍乡市', '营口市', '葫芦岛市', '蚌埠市', '衡水市', '衡阳市', '衢州市', '襄阳市', - '西宁市', '西安市', '许昌市', '贵港市', '贵阳市', '贺州市', '资阳市', '赣州市', '赤峰市', - '辽源市', '辽阳市', '达州市', '运城市', '连云港市', '通化市', '通辽市', '遂宁市', '遵义市', - '邢台市', '那曲地区', '邯郸市', '邵阳市', '郑州市', '郴州市', '鄂尔多斯市', '鄂州市', '酒泉市', + '神农架林区', '福州市', '秦皇岛市', '红河哈尼族彝族自治州', '绥化市', '邢台市', '那曲地区', '邯郸市', '邵阳市', '郑州市', '郴州市', '鄂尔多斯市', '鄂州市', '酒泉市', '重庆市', '金华市', '金昌市', '钦州市', '铁岭市', '铜仁地区', '铜川市', '铜陵市', '银川市', '锡林郭勒盟', '锦州市', '镇江市', '长春市', '长沙市', '长治市', '阜新市', '阜阳市', '防城港市', '阳江市', '阳泉市', '阿克苏地区', '阿勒泰地区', '阿拉善盟', '阿拉尔市', '阿里地区', '陇南市', @@ -169,7 +165,9 @@ class GaoDePoiDataProcess extends BaseBatch{ '鸡西市', '鹤壁市', '鹤岗市', '鹰潭市', '黄冈市', '黄山市', '黄石市', '黑河市', '黔东南苗族侗族自治州', '黔南布依族苗族自治州', '黔西南布依族苗族自治州', '齐齐哈尔市', '龙岩市'] - def runbuducitys = [] + def runbuducitys = ["湖州市","宁波市","丽水市","杭州市","温州市","嘉兴市","台州市","绍兴市","金华市","舟山市","衢州市", + '绍兴市','七台河市','万宁市', '三亚市', '三明市', '三门峡市', '上饶市', '东方市', '东莞市','上海市', + '东营市', '中卫市'] for(def city : buducitys){ if(runbuducitys.contains(city)){ continue @@ -180,7 +178,7 @@ class GaoDePoiDataProcess extends BaseBatch{ save(city, rows) // saveToMG(city, rows) - println city +" "+pois.size()+" add:"+rows.size() + println city +" "+pois.size()+" add:"+rows.size()+" "+new Date().format("yyyy-MM-dd HH:mm:ss") } } @@ -195,7 +193,7 @@ class GaoDePoiDataProcess extends BaseBatch{ def poiKeysFilter = findEsPoiByCity(city) println "Es 加载完成:"+new Date().format("yyyy-MM-dd HH:mm:ss") for(def i=0;i Date: Fri, 23 Aug 2024 10:06:04 +0800 Subject: [PATCH 6/6] =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/gj/xiamen/GaoDePoiDataProcess.groovy | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy index eacf43517..95c0a4d0d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy @@ -32,6 +32,7 @@ import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilders import java.nio.charset.StandardCharsets +import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService @@ -184,8 +185,8 @@ class GaoDePoiDataProcess extends BaseBatch{ } def createVillageData(city, pois){ - def rets = new CopyOnWriteArrayList() - def keyRets = new CopyOnWriteArrayList() + def rets = new ConcurrentHashMap() + //def keyRets = new CopyOnWriteArrayList() def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(pois.size()) // def townMap = findTownRange(city) @@ -204,9 +205,10 @@ class GaoDePoiDataProcess extends BaseBatch{ def prov = ProvinceData.getProvince(p.city.replace("市","")) def ret = createVillageData(prov, p, townMap) - if(!keyRets.contains(ret.city+"##"+ret.district+"##"+ret.name)){ - keyRets << ret.city+"##"+ret.district+"##"+ret.name - rets << ret + if(!rets.containsKey(ret.city+"##"+ret.district+"##"+ret.name)){ + //keyRets << ret.city+"##"+ret.district+"##"+ret.name + //rets << ret + rets.put(ret.city+"##"+ret.district+"##"+ret.name, ret) } } @@ -693,8 +695,8 @@ class GaoDePoiDataProcess extends BaseBatch{ int batchSize = 1000; int processedCount = 0; def client = ElasticUtils89.getClient("192.168.0.89") - for (int i = 0; i < rows.size(); i++) { - def map = rows[i] + for (String key : rows.keySet()) { + def map = rows.get(key) request.add(new IndexRequest("didata_map_poi_single", "_doc", map.id).source(map, XContentType.JSON)); if(i%1000==0){ print "."