+1
-1
@@ -4209,7 +4209,7 @@ class CommonDatautils {
|
||||
|
||||
|
||||
static def findAllNewPoiDatas(city){
|
||||
String sql = "select pname,cityname,adname,name,address,location,type,id from poi_amap_2024 where cityname='${city}' "
|
||||
String sql = "select pname,cityname,adname,name,address,location,type,id from poi_amap_2024 where cityname='${city}'"
|
||||
def rets = []
|
||||
def index= 0
|
||||
ElasticUtils.executeSqlQuery(sql, { map->
|
||||
|
||||
+24
-11
@@ -71,7 +71,7 @@ class NewGaoDePoiBatch extends BaseBatch{
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(results.size())
|
||||
def townMap = CommonDatautils.queryFullRegionTownByCity(city, ["city","district","street", "shape"])
|
||||
def townMap = CommonDatautils.queryTownByCity(city, ["city","district","mzb_district_name", "street", "shape","alias"])
|
||||
for(def i=0;i<results.size();i++){
|
||||
def map = results[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
@@ -93,9 +93,15 @@ class NewGaoDePoiBatch extends BaseBatch{
|
||||
|
||||
def createData(map, townMap){
|
||||
def saveObj = [:]
|
||||
def towns = townMap.get(saveObj.district)
|
||||
saveObj.province = map.province
|
||||
saveObj.city = map.city
|
||||
saveObj.district = map.district
|
||||
if(towns){
|
||||
saveObj.district = map.district
|
||||
}else{
|
||||
println map.city +" "+map.district +" 格式化不出区域"
|
||||
}
|
||||
|
||||
saveObj.name = map.name
|
||||
if(map.cat){
|
||||
def arrs = map.cat.split(";|:")
|
||||
@@ -142,7 +148,6 @@ class NewGaoDePoiBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
if(location){
|
||||
def towns = townMap.get(saveObj.district)
|
||||
if(towns){
|
||||
for(def town : towns){
|
||||
if(town.shape && town.shape.coordinates){
|
||||
@@ -153,15 +158,23 @@ class NewGaoDePoiBatch extends BaseBatch{
|
||||
}
|
||||
}
|
||||
}
|
||||
if(!saveObj.township && map.address.contains("街道")){
|
||||
println map.address +" 含街道"
|
||||
if(map.address && saveObj.township){
|
||||
for(def town : towns){
|
||||
if(map.address.contains(town.street)){
|
||||
saveObj.township = town.street
|
||||
break
|
||||
}
|
||||
}
|
||||
if(map.address && (map.address.contains("街道") || map.address.contains("镇") || map.address.contains("乡"))){
|
||||
println map.address +" 含街道,镇,乡"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(map.address){
|
||||
if(map.address.length()<3){
|
||||
saveObj.township = ""
|
||||
saveObj.address = ""
|
||||
}else{
|
||||
def address = map.address
|
||||
def i = 0
|
||||
@@ -172,18 +185,18 @@ class NewGaoDePoiBatch extends BaseBatch{
|
||||
if(address.toString().startsWith(map.city)){
|
||||
address = address.toString().substring(0, map.city.toString().length())
|
||||
}
|
||||
if(address.toString().startsWith(map.district)){
|
||||
if(map.district && address.toString().startsWith(map.district)){
|
||||
address = address.toString().substring(0, map.district.toString().length())
|
||||
}
|
||||
if(address.toString().startsWith("市辖区")){
|
||||
address = address.toString().substring(0, 3)
|
||||
}
|
||||
if(address.toString().startsWith(saveObj.township)){
|
||||
if(saveObj.township && address.toString().startsWith(saveObj.township)){
|
||||
address = address.toString().substring(0, saveObj.township.toString().length())
|
||||
}
|
||||
i++
|
||||
}
|
||||
saveObj.township = address
|
||||
saveObj.address = address
|
||||
}
|
||||
}
|
||||
return saveObj
|
||||
@@ -195,10 +208,10 @@ class NewGaoDePoiBatch extends BaseBatch{
|
||||
int processedCount = 0;
|
||||
def client = ElasticUtils89.getClient("192.168.0.89")
|
||||
|
||||
rows.each{key,map->
|
||||
rows.each{map->
|
||||
// def map = rows.get(key)
|
||||
|
||||
request.add(new IndexRequest("didata_map_poi_single", "_doc", map.id).source(map, XContentType.JSON));
|
||||
request.add(new IndexRequest("didata_map_poi_2024", "_doc", map.data_id).source(map, XContentType.JSON));
|
||||
|
||||
// 批量提交数据
|
||||
if (request.numberOfActions() >= batchSize) {
|
||||
|
||||
-1
@@ -46,7 +46,6 @@ class RegionStandardBatch extends BaseBatch{
|
||||
for(def city : citys){
|
||||
index ++
|
||||
|
||||
|
||||
if(runCitys.contains(city)){
|
||||
continue
|
||||
}
|
||||
|
||||
+30
-12
@@ -24,6 +24,7 @@ class TownStandardBatch extends BaseBatch{
|
||||
def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity()
|
||||
|
||||
def updateList = []
|
||||
def addAliasList = []
|
||||
|
||||
static main(args) {
|
||||
runBatch(new TownStandardBatch(), args)
|
||||
@@ -66,7 +67,10 @@ class TownStandardBatch extends BaseBatch{
|
||||
|
||||
def townStandardByPoi(){
|
||||
updateList << ["数据ID", "城市","区域","乡镇街道","名称","状态", "修改"]
|
||||
def citys = ["福州"]//CommonDatautils.findAllCity()
|
||||
|
||||
addAliasList << ["城市","区域","乡镇街道", "新增别名"]
|
||||
//浙江,江苏,安徽
|
||||
def citys = ['杭州','宁波','温州','嘉兴','台州','湖州','绍兴','金华','舟山','衢州','丽水']//CommonDatautils.findAllCity()
|
||||
for(def city : citys){
|
||||
|
||||
def data = fullCityMap.get(city)
|
||||
@@ -78,7 +82,7 @@ class TownStandardBatch extends BaseBatch{
|
||||
|
||||
def fileName = "/home/goojia/gj/poi/${fullCityName}.csv"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\${fullCityName}.csv"
|
||||
fileName = "D:\\数据\\poi\\${fullCityName}.csv"
|
||||
}
|
||||
File file=new File(fileName);
|
||||
if(file.exists()){
|
||||
@@ -104,6 +108,8 @@ class TownStandardBatch extends BaseBatch{
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\乡镇街道待查数据.xlsx", updateList, "数据")
|
||||
ExcelUtils.write("D:\\数据\\乡镇街道待查数据-需新增别名.xlsx", addAliasList, "数据")
|
||||
|
||||
}
|
||||
def haveRegionProcess(city, fullCityName, townMap, cat, shortRegionNameMap){
|
||||
def results = []
|
||||
@@ -196,6 +202,18 @@ class TownStandardBatch extends BaseBatch{
|
||||
}
|
||||
break
|
||||
}
|
||||
// else{
|
||||
// if(town.street.contains(d.township.replace("镇","").replace("街道","").replace("乡",""))){
|
||||
// updateTown = town.street
|
||||
// opt = false
|
||||
// def addObj = [town.city, town.district, town.street, d.township]
|
||||
// if(!addAliasList.contains(addObj)){
|
||||
// addAliasList << addObj
|
||||
// }
|
||||
//
|
||||
// break
|
||||
// }
|
||||
// }
|
||||
}
|
||||
}
|
||||
if(opt){
|
||||
@@ -203,12 +221,12 @@ class TownStandardBatch extends BaseBatch{
|
||||
def m = [:]
|
||||
m.put(d.id, ["township":updateTown])
|
||||
results << m
|
||||
if(results.size()%100==0){
|
||||
updateByEs(results)
|
||||
results.clear()
|
||||
}
|
||||
// if(results.size()%100==0){
|
||||
// updateByEs(results)
|
||||
// results.clear()
|
||||
// }
|
||||
updateList << [d.id, d.city, d.district, d.name,d.township, d.status, updateTown]
|
||||
println d.city +" "+d.district+" "+d.name +" "+d.township+" 乡镇修改为:"+updateTown
|
||||
// println d.city +" "+d.district+" "+d.name +" "+d.township+" 乡镇修改为:"+updateTown
|
||||
}else{
|
||||
def m = [:]
|
||||
m.put(d.id, ["township":""])
|
||||
@@ -226,10 +244,10 @@ class TownStandardBatch extends BaseBatch{
|
||||
}
|
||||
}
|
||||
|
||||
if(results.size()>0){
|
||||
updateByEs(results)
|
||||
results.clear()
|
||||
}
|
||||
// if(results.size()>0){
|
||||
// updateByEs(results)
|
||||
// results.clear()
|
||||
// }
|
||||
println fullCityName +" 处理完成"
|
||||
}
|
||||
|
||||
@@ -258,7 +276,7 @@ class TownStandardBatch extends BaseBatch{
|
||||
println("批量更新中有失败的操作:")
|
||||
bulkResponse.buildFailureMessage().each { println(it) }
|
||||
} else {
|
||||
println("批量更新成功!")
|
||||
// println("批量更新成功!")
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
|
||||
Reference in New Issue
Block a user