From d5b873552397af58fd176e36f158279429e6f2b3 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 26 Nov 2024 14:44:55 +0800 Subject: [PATCH] update --- .../script/commData/CommonDatautils.groovy | 2 +- .../gj/data_handle/NewGaoDePoiBatch.groovy | 35 +++++++++++----- .../gj/data_handle/RegionStandardBatch.groovy | 1 - .../gj/data_handle/TownStandardBatch.groovy | 42 +++++++++++++------ 4 files changed, 55 insertions(+), 25 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 033959ead..c3d473a3b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -4209,7 +4209,7 @@ class CommonDatautils { static def findAllNewPoiDatas(city){ - String sql = "select pname,cityname,adname,name,address,location,type,id from poi_amap_2024 where cityname='${city}' " + String sql = "select pname,cityname,adname,name,address,location,type,id from poi_amap_2024 where cityname='${city}'" def rets = [] def index= 0 ElasticUtils.executeSqlQuery(sql, { map-> diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy index 8fc228e3d..bd1f5467a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy @@ -71,7 +71,7 @@ class NewGaoDePoiBatch extends BaseBatch{ def rets = new CopyOnWriteArrayList() def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(results.size()) - def townMap = CommonDatautils.queryFullRegionTownByCity(city, ["city","district","street", "shape"]) + def townMap = CommonDatautils.queryTownByCity(city, ["city","district","mzb_district_name", "street", "shape","alias"]) for(def i=0;i + rows.each{map-> // def map = rows.get(key) - request.add(new IndexRequest("didata_map_poi_single", "_doc", map.id).source(map, XContentType.JSON)); + request.add(new IndexRequest("didata_map_poi_2024", "_doc", map.data_id).source(map, XContentType.JSON)); // 批量提交数据 if (request.numberOfActions() >= batchSize) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy index 0afba947a..abce57fe0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy @@ -46,7 +46,6 @@ class RegionStandardBatch extends BaseBatch{ for(def city : citys){ index ++ - if(runCitys.contains(city)){ continue } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy index 545a0d186..4d3e00e32 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy @@ -24,6 +24,7 @@ class TownStandardBatch extends BaseBatch{ def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity() def updateList = [] + def addAliasList = [] static main(args) { runBatch(new TownStandardBatch(), args) @@ -66,7 +67,10 @@ class TownStandardBatch extends BaseBatch{ def townStandardByPoi(){ updateList << ["数据ID", "城市","区域","乡镇街道","名称","状态", "修改"] - def citys = ["福州"]//CommonDatautils.findAllCity() + + addAliasList << ["城市","区域","乡镇街道", "新增别名"] + //浙江,江苏,安徽 + def citys = ['杭州','宁波','温州','嘉兴','台州','湖州','绍兴','金华','舟山','衢州','丽水']//CommonDatautils.findAllCity() for(def city : citys){ def data = fullCityMap.get(city) @@ -78,7 +82,7 @@ class TownStandardBatch extends BaseBatch{ def fileName = "/home/goojia/gj/poi/${fullCityName}.csv" if (CommonDatautils.isWindow()) { - fileName = "D:\\数据\\${fullCityName}.csv" + fileName = "D:\\数据\\poi\\${fullCityName}.csv" } File file=new File(fileName); if(file.exists()){ @@ -104,6 +108,8 @@ class TownStandardBatch extends BaseBatch{ } } ExcelUtils.write("D:\\数据\\乡镇街道待查数据.xlsx", updateList, "数据") + ExcelUtils.write("D:\\数据\\乡镇街道待查数据-需新增别名.xlsx", addAliasList, "数据") + } def haveRegionProcess(city, fullCityName, townMap, cat, shortRegionNameMap){ def results = [] @@ -196,6 +202,18 @@ class TownStandardBatch extends BaseBatch{ } break } +// else{ +// if(town.street.contains(d.township.replace("镇","").replace("街道","").replace("乡",""))){ +// updateTown = town.street +// opt = false +// def addObj = [town.city, town.district, town.street, d.township] +// if(!addAliasList.contains(addObj)){ +// addAliasList << addObj +// } +// +// break +// } +// } } } if(opt){ @@ -203,12 +221,12 @@ class TownStandardBatch extends BaseBatch{ def m = [:] m.put(d.id, ["township":updateTown]) results << m - if(results.size()%100==0){ - updateByEs(results) - results.clear() - } +// if(results.size()%100==0){ +// updateByEs(results) +// results.clear() +// } updateList << [d.id, d.city, d.district, d.name,d.township, d.status, updateTown] - println d.city +" "+d.district+" "+d.name +" "+d.township+" 乡镇修改为:"+updateTown +// println d.city +" "+d.district+" "+d.name +" "+d.township+" 乡镇修改为:"+updateTown }else{ def m = [:] m.put(d.id, ["township":""]) @@ -226,10 +244,10 @@ class TownStandardBatch extends BaseBatch{ } } - if(results.size()>0){ - updateByEs(results) - results.clear() - } +// if(results.size()>0){ +// updateByEs(results) +// results.clear() +// } println fullCityName +" 处理完成" } @@ -258,7 +276,7 @@ class TownStandardBatch extends BaseBatch{ println("批量更新中有失败的操作:") bulkResponse.buildFailureMessage().each { println(it) } } else { - println("批量更新成功!") +// println("批量更新成功!") } } catch (Exception e) { e.printStackTrace()