Merge remote-tracking branch 'origin/master'

This commit is contained in:
sjshenjian
2024-11-26 20:19:52 +08:00
2 changed files with 109 additions and 33 deletions
@@ -4213,9 +4213,11 @@ class CommonDatautils {
def rets = [] def rets = []
def index= 0 def index= 0
ElasticUtils.executeSqlQuery(sql, { map-> ElasticUtils.executeSqlQuery(sql, { map->
index++ // if(map.address && map.address.contains("五角场")){
println index index++
rets << ["map_id":map.id,"province":map.pname,"city":map.cityname, "district":map.adname, "name":map.name, "address":map.address, "cat":map.type,"lng":map.location.x, "lat":map.location.y] println index
rets << ["map_id":map.id,"province":map.pname,"city":map.cityname, "district":map.adname, "name":map.name, "address":map.address, "cat":map.type,"lng":map.location.x, "lat":map.location.y]
// }
}) })
return rets; return rets;
} }
@@ -29,6 +29,8 @@ class NewGaoDePoiBatch extends BaseBatch{
public static def version = new Date().getTime() public static def version = new Date().getTime()
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity()
public static def TAGS = [] public static def TAGS = []
static main(args) { static main(args) {
@@ -50,12 +52,12 @@ class NewGaoDePoiBatch extends BaseBatch{
def cityList = CommonDatautils.findFullNameCitys().reverse() def cityList = CommonDatautils.findFullNameCitys().reverse()
for(def cityObj : cityList){ for(def cityObj : cityList){
if(cityObj.city == "上海市"){ // if(cityObj.city == "上海市"){
def results = CommonDatautils.findAllNewPoiDatas(cityObj.city) def results = CommonDatautils.findAllNewPoiDatas(cityObj.city)
println cityObj.city+" "+results.size() println cityObj.city+" "+results.size()
def rows = running(cityObj.city, results) def rows = running(cityObj.city, results)
save(rows) save(rows)
} // }
} }
} }
def getAllPoiType(filePath){ def getAllPoiType(filePath){
@@ -72,13 +74,14 @@ class NewGaoDePoiBatch extends BaseBatch{
def totalCount = new AtomicLong() def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(results.size()) final CountDownLatch latch = new CountDownLatch(results.size())
def townMap = CommonDatautils.queryTownByCity(city, ["city","district","mzb_district_name", "street", "shape","alias"]) def townMap = CommonDatautils.queryTownByCity(city, ["city","district","mzb_district_name", "street", "shape","alias"])
def mzbRegionMap = regionFullNameMap.get(city)
for(def i=0;i<results.size();i++){ for(def i=0;i<results.size();i++){
def map = results[i] def map = results[i]
fixedThreadPool.execute(new Runnable() { fixedThreadPool.execute(new Runnable() {
void run() { void run() {
try{ try{
totalCount.addAndGet(1) totalCount.addAndGet(1)
rets.addAll(createData(map, townMap)) rets.addAll(createData(map, mzbRegionMap, townMap))
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace() e.printStackTrace()
} finally { } finally {
@@ -91,13 +94,37 @@ class NewGaoDePoiBatch extends BaseBatch{
return rets; return rets;
} }
def createData(map, townMap){ def getStandardRegion(mzbRegionMap, district){
def tjjObj = mzbRegionMap.get(district)
if(tjjObj && tjjObj.mzb_district_name && tjjObj.mzb_district_name == district){
return district
}
def updateRegion = null
if(tjjObj){
if(tjjObj.mzb_district_name && tjjObj.mzb_district_name!=district){
updateRegion = tjjObj.mzb_district_name
}
}else{
def set = mzbRegionMap.keySet()
for(def key : set){
def tempObj = mzbRegionMap.get(key)
def alias = tempObj.alias
if(alias.contains(district) || district==tempObj.mzb_district_name){
updateRegion = key
break;
}
}
}
return updateRegion
}
def createData(map, mzbRegionMap, townMap){
def saveObj = [:] def saveObj = [:]
def towns = townMap.get(map.district)
saveObj.province = map.province saveObj.province = map.province
saveObj.city = map.city saveObj.city = map.city
if(towns){ def newDistrict = getStandardRegion(mzbRegionMap, map.district)
saveObj.district = map.district if(newDistrict){
saveObj.district = newDistrict
}else{ }else{
println map.city +" "+map.district +" 格式化不出区域" println map.city +" "+map.district +" 格式化不出区域"
} }
@@ -147,6 +174,7 @@ class NewGaoDePoiBatch extends BaseBatch{
saveObj.status = 1 saveObj.status = 1
} }
def towns = townMap.get(saveObj.district)
if(location){ if(location){
if(towns){ if(towns){
for(def town : towns){ for(def town : towns){
@@ -158,17 +186,17 @@ class NewGaoDePoiBatch extends BaseBatch{
} }
} }
} }
if(map.address && !saveObj.township){ // if(map.address && !saveObj.township){
for(def town : towns){ // for(def town : towns){
if(map.address.contains(town.street)){ // if(map.address.contains(town.street)){
saveObj.township = town.street // saveObj.township = town.street
break // break
} // }
} // }
if(!saveObj.township && map.address && (map.address.contains("街道") || map.address.contains("镇") || map.address.contains("乡"))){ // if(!saveObj.township && map.address && (map.address.contains("街道") || map.address.contains("镇") || map.address.contains("乡"))){
println map.address +" 含街道,镇,乡" // println map.address +" 含街道,镇,乡"
} // }
} // }
} }
} }
@@ -179,20 +207,68 @@ class NewGaoDePoiBatch extends BaseBatch{
def address = map.address def address = map.address
def i = 0 def i = 0
while(i<5){ while(i<5){
if(address.toString().startsWith(map.province)){ if(address && address.toString().startsWith(map.province)){
address = address.toString().substring(0, map.province.toString().length()) address = address.toString().substring(map.province.toString().length())
} }
if(address.toString().startsWith(map.city)){ if(address && address.toString().startsWith(map.city)){
address = address.toString().substring(0, map.city.toString().length()) address = address.toString().substring(map.city.toString().length())
} }
if(map.district && address.toString().startsWith(map.district)){
address = address.toString().substring(0, map.district.toString().length()) if(address && saveObj.district){
def tjjObj = mzbRegionMap.get(saveObj.district)
if(tjjObj){
def alias = tjjObj.alias
if(!alias){
alias = []
}
if(!alias.contains(saveObj.district)){
alias << saveObj.district
}
// if(alias.size()>=2){
// alias.sort(){ a, b ->
// b.length().compareTo(a.length());
// }
// }
for(def regionName : alias){
if(address.startsWith(regionName) && (regionName.endsWith("区") || regionName.endsWith("县"))){
address = address.substring(regionName.toString().length())
break
}
}
}
} }
if(address.toString().startsWith("市辖区")){
address = address.toString().substring(0, 3)
if(address && address.toString().startsWith("市辖区")){
address = address.toString().substring(3)
} }
if(saveObj.township && address.toString().startsWith(saveObj.township)){
address = address.toString().substring(0, saveObj.township.toString().length()) if(address && towns){
for(def town : towns){
def townAlias = town.alias
if(!townAlias){
townAlias = []
}
if(!townAlias.contains(town.street)){
townAlias << town.street
}
// if(townAlias.size()>=2){
// townAlias = townAlias.sort(){ a, b ->
// b.length().compareTo(a.length());
// }
// }
for(def townName : townAlias){
if(address.startsWith(townName)){
address = address.substring(townName.toString().length())
// if(!saveObj.township){
// saveObj.township = town.street
// }
break
}
}
}
} }
i++ i++
} }
@@ -201,8 +277,6 @@ class NewGaoDePoiBatch extends BaseBatch{
}else{ }else{
saveObj.address = address saveObj.address = address
} }
} }
} }
return saveObj return saveObj