Merge branch 'script_gj' into 'master'

update

See merge request !1021
This commit is contained in:
2024-11-26 14:38:06 +08:00
4 changed files with 55 additions and 25 deletions
@@ -4209,7 +4209,7 @@ class CommonDatautils {
static def findAllNewPoiDatas(city){
String sql = "select pname,cityname,adname,name,address,location,type,id from poi_amap_2024 where cityname='${city}' "
String sql = "select pname,cityname,adname,name,address,location,type,id from poi_amap_2024 where cityname='${city}'"
def rets = []
def index= 0
ElasticUtils.executeSqlQuery(sql, { map->
@@ -71,7 +71,7 @@ class NewGaoDePoiBatch extends BaseBatch{
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(results.size())
def townMap = CommonDatautils.queryFullRegionTownByCity(city, ["city","district","street", "shape"])
def townMap = CommonDatautils.queryTownByCity(city, ["city","district","mzb_district_name", "street", "shape","alias"])
for(def i=0;i<results.size();i++){
def map = results[i]
fixedThreadPool.execute(new Runnable() {
@@ -93,9 +93,15 @@ class NewGaoDePoiBatch extends BaseBatch{
def createData(map, townMap){
def saveObj = [:]
def towns = townMap.get(saveObj.district)
saveObj.province = map.province
saveObj.city = map.city
saveObj.district = map.district
if(towns){
saveObj.district = map.district
}else{
println map.city +" "+map.district +" 格式化不出区域"
}
saveObj.name = map.name
if(map.cat){
def arrs = map.cat.split(";|:")
@@ -142,7 +148,6 @@ class NewGaoDePoiBatch extends BaseBatch{
}
if(location){
def towns = townMap.get(saveObj.district)
if(towns){
for(def town : towns){
if(town.shape && town.shape.coordinates){
@@ -153,15 +158,23 @@ class NewGaoDePoiBatch extends BaseBatch{
}
}
}
if(!saveObj.township && map.address.contains("街道")){
println map.address +" 含街道"
if(map.address && saveObj.township){
for(def town : towns){
if(map.address.contains(town.street)){
saveObj.township = town.street
break
}
}
if(map.address && (map.address.contains("街道") || map.address.contains("镇") || map.address.contains("乡"))){
println map.address +" 含街道,镇,乡"
}
}
}
}
if(map.address){
if(map.address.length()<3){
saveObj.township = ""
saveObj.address = ""
}else{
def address = map.address
def i = 0
@@ -172,18 +185,18 @@ class NewGaoDePoiBatch extends BaseBatch{
if(address.toString().startsWith(map.city)){
address = address.toString().substring(0, map.city.toString().length())
}
if(address.toString().startsWith(map.district)){
if(map.district && address.toString().startsWith(map.district)){
address = address.toString().substring(0, map.district.toString().length())
}
if(address.toString().startsWith("市辖区")){
address = address.toString().substring(0, 3)
}
if(address.toString().startsWith(saveObj.township)){
if(saveObj.township && address.toString().startsWith(saveObj.township)){
address = address.toString().substring(0, saveObj.township.toString().length())
}
i++
}
saveObj.township = address
saveObj.address = address
}
}
return saveObj
@@ -195,10 +208,10 @@ class NewGaoDePoiBatch extends BaseBatch{
int processedCount = 0;
def client = ElasticUtils89.getClient("192.168.0.89")
rows.each{key,map->
rows.each{map->
// def map = rows.get(key)
request.add(new IndexRequest("didata_map_poi_single", "_doc", map.id).source(map, XContentType.JSON));
request.add(new IndexRequest("didata_map_poi_2024", "_doc", map.data_id).source(map, XContentType.JSON));
// 批量提交数据
if (request.numberOfActions() >= batchSize) {
@@ -46,7 +46,6 @@ class RegionStandardBatch extends BaseBatch{
for(def city : citys){
index ++
if(runCitys.contains(city)){
continue
}
@@ -24,6 +24,7 @@ class TownStandardBatch extends BaseBatch{
def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity()
def updateList = []
def addAliasList = []
static main(args) {
runBatch(new TownStandardBatch(), args)
@@ -66,7 +67,10 @@ class TownStandardBatch extends BaseBatch{
def townStandardByPoi(){
updateList << ["数据ID", "城市","区域","乡镇街道","名称","状态", "修改"]
def citys = ["福州"]//CommonDatautils.findAllCity()
addAliasList << ["城市","区域","乡镇街道", "新增别名"]
//浙江,江苏,安徽
def citys = ['杭州','宁波','温州','嘉兴','台州','湖州','绍兴','金华','舟山','衢州','丽水']//CommonDatautils.findAllCity()
for(def city : citys){
def data = fullCityMap.get(city)
@@ -78,7 +82,7 @@ class TownStandardBatch extends BaseBatch{
def fileName = "/home/goojia/gj/poi/${fullCityName}.csv"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\${fullCityName}.csv"
fileName = "D:\\数据\\poi\\${fullCityName}.csv"
}
File file=new File(fileName);
if(file.exists()){
@@ -104,6 +108,8 @@ class TownStandardBatch extends BaseBatch{
}
}
ExcelUtils.write("D:\\数据\\乡镇街道待查数据.xlsx", updateList, "数据")
ExcelUtils.write("D:\\数据\\乡镇街道待查数据-需新增别名.xlsx", addAliasList, "数据")
}
def haveRegionProcess(city, fullCityName, townMap, cat, shortRegionNameMap){
def results = []
@@ -196,6 +202,18 @@ class TownStandardBatch extends BaseBatch{
}
break
}
// else{
// if(town.street.contains(d.township.replace("镇","").replace("街道","").replace("乡",""))){
// updateTown = town.street
// opt = false
// def addObj = [town.city, town.district, town.street, d.township]
// if(!addAliasList.contains(addObj)){
// addAliasList << addObj
// }
//
// break
// }
// }
}
}
if(opt){
@@ -203,12 +221,12 @@ class TownStandardBatch extends BaseBatch{
def m = [:]
m.put(d.id, ["township":updateTown])
results << m
if(results.size()%100==0){
updateByEs(results)
results.clear()
}
// if(results.size()%100==0){
// updateByEs(results)
// results.clear()
// }
updateList << [d.id, d.city, d.district, d.name,d.township, d.status, updateTown]
println d.city +" "+d.district+" "+d.name +" "+d.township+" 乡镇修改为:"+updateTown
// println d.city +" "+d.district+" "+d.name +" "+d.township+" 乡镇修改为:"+updateTown
}else{
def m = [:]
m.put(d.id, ["township":""])
@@ -226,10 +244,10 @@ class TownStandardBatch extends BaseBatch{
}
}
if(results.size()>0){
updateByEs(results)
results.clear()
}
// if(results.size()>0){
// updateByEs(results)
// results.clear()
// }
println fullCityName +" 处理完成"
}
@@ -258,7 +276,7 @@ class TownStandardBatch extends BaseBatch{
println("批量更新中有失败的操作:")
bulkResponse.buildFailureMessage().each { println(it) }
} else {
println("批量更新成功!")
// println("批量更新成功!")
}
} catch (Exception e) {
e.printStackTrace()