Merge remote-tracking branch 'origin/master'

This commit is contained in:
sjshenjian
2024-11-26 20:19:52 +08:00
2 changed files with 109 additions and 33 deletions
@@ -4213,9 +4213,11 @@ class CommonDatautils {
def rets = []
def index= 0
ElasticUtils.executeSqlQuery(sql, { map->
index++
println index
rets << ["map_id":map.id,"province":map.pname,"city":map.cityname, "district":map.adname, "name":map.name, "address":map.address, "cat":map.type,"lng":map.location.x, "lat":map.location.y]
// if(map.address && map.address.contains("五角场")){
index++
println index
rets << ["map_id":map.id,"province":map.pname,"city":map.cityname, "district":map.adname, "name":map.name, "address":map.address, "cat":map.type,"lng":map.location.x, "lat":map.location.y]
// }
})
return rets;
}
@@ -29,6 +29,8 @@ class NewGaoDePoiBatch extends BaseBatch{
public static def version = new Date().getTime()
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity()
public static def TAGS = []
static main(args) {
@@ -50,12 +52,12 @@ class NewGaoDePoiBatch extends BaseBatch{
def cityList = CommonDatautils.findFullNameCitys().reverse()
for(def cityObj : cityList){
if(cityObj.city == "上海市"){
// if(cityObj.city == "上海市"){
def results = CommonDatautils.findAllNewPoiDatas(cityObj.city)
println cityObj.city+" "+results.size()
def rows = running(cityObj.city, results)
save(rows)
}
// }
}
}
def getAllPoiType(filePath){
@@ -72,13 +74,14 @@ class NewGaoDePoiBatch extends BaseBatch{
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(results.size())
def townMap = CommonDatautils.queryTownByCity(city, ["city","district","mzb_district_name", "street", "shape","alias"])
def mzbRegionMap = regionFullNameMap.get(city)
for(def i=0;i<results.size();i++){
def map = results[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
rets.addAll(createData(map, townMap))
rets.addAll(createData(map, mzbRegionMap, townMap))
} catch (Exception e) {
e.printStackTrace()
} finally {
@@ -91,13 +94,37 @@ class NewGaoDePoiBatch extends BaseBatch{
return rets;
}
def createData(map, townMap){
def getStandardRegion(mzbRegionMap, district){
def tjjObj = mzbRegionMap.get(district)
if(tjjObj && tjjObj.mzb_district_name && tjjObj.mzb_district_name == district){
return district
}
def updateRegion = null
if(tjjObj){
if(tjjObj.mzb_district_name && tjjObj.mzb_district_name!=district){
updateRegion = tjjObj.mzb_district_name
}
}else{
def set = mzbRegionMap.keySet()
for(def key : set){
def tempObj = mzbRegionMap.get(key)
def alias = tempObj.alias
if(alias.contains(district) || district==tempObj.mzb_district_name){
updateRegion = key
break;
}
}
}
return updateRegion
}
def createData(map, mzbRegionMap, townMap){
def saveObj = [:]
def towns = townMap.get(map.district)
saveObj.province = map.province
saveObj.city = map.city
if(towns){
saveObj.district = map.district
def newDistrict = getStandardRegion(mzbRegionMap, map.district)
if(newDistrict){
saveObj.district = newDistrict
}else{
println map.city +" "+map.district +" 格式化不出区域"
}
@@ -147,6 +174,7 @@ class NewGaoDePoiBatch extends BaseBatch{
saveObj.status = 1
}
def towns = townMap.get(saveObj.district)
if(location){
if(towns){
for(def town : towns){
@@ -158,17 +186,17 @@ class NewGaoDePoiBatch extends BaseBatch{
}
}
}
if(map.address && !saveObj.township){
for(def town : towns){
if(map.address.contains(town.street)){
saveObj.township = town.street
break
}
}
if(!saveObj.township && map.address && (map.address.contains("街道") || map.address.contains("镇") || map.address.contains("乡"))){
println map.address +" 含街道,镇,乡"
}
}
// if(map.address && !saveObj.township){
// for(def town : towns){
// if(map.address.contains(town.street)){
// saveObj.township = town.street
// break
// }
// }
// if(!saveObj.township && map.address && (map.address.contains("街道") || map.address.contains("镇") || map.address.contains("乡"))){
// println map.address +" 含街道,镇,乡"
// }
// }
}
}
@@ -179,20 +207,68 @@ class NewGaoDePoiBatch extends BaseBatch{
def address = map.address
def i = 0
while(i<5){
if(address.toString().startsWith(map.province)){
address = address.toString().substring(0, map.province.toString().length())
if(address && address.toString().startsWith(map.province)){
address = address.toString().substring(map.province.toString().length())
}
if(address.toString().startsWith(map.city)){
address = address.toString().substring(0, map.city.toString().length())
if(address && address.toString().startsWith(map.city)){
address = address.toString().substring(map.city.toString().length())
}
if(map.district && address.toString().startsWith(map.district)){
address = address.toString().substring(0, map.district.toString().length())
if(address && saveObj.district){
def tjjObj = mzbRegionMap.get(saveObj.district)
if(tjjObj){
def alias = tjjObj.alias
if(!alias){
alias = []
}
if(!alias.contains(saveObj.district)){
alias << saveObj.district
}
// if(alias.size()>=2){
// alias.sort(){ a, b ->
// b.length().compareTo(a.length());
// }
// }
for(def regionName : alias){
if(address.startsWith(regionName) && (regionName.endsWith("区") || regionName.endsWith("县"))){
address = address.substring(regionName.toString().length())
break
}
}
}
}
if(address.toString().startsWith("市辖区")){
address = address.toString().substring(0, 3)
if(address && address.toString().startsWith("市辖区")){
address = address.toString().substring(3)
}
if(saveObj.township && address.toString().startsWith(saveObj.township)){
address = address.toString().substring(0, saveObj.township.toString().length())
if(address && towns){
for(def town : towns){
def townAlias = town.alias
if(!townAlias){
townAlias = []
}
if(!townAlias.contains(town.street)){
townAlias << town.street
}
// if(townAlias.size()>=2){
// townAlias = townAlias.sort(){ a, b ->
// b.length().compareTo(a.length());
// }
// }
for(def townName : townAlias){
if(address.startsWith(townName)){
address = address.substring(townName.toString().length())
// if(!saveObj.township){
// saveObj.township = town.street
// }
break
}
}
}
}
i++
}
@@ -201,8 +277,6 @@ class NewGaoDePoiBatch extends BaseBatch{
}else{
saveObj.address = address
}
}
}
return saveObj