diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index ed3314d22..9db36e3d8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -4215,7 +4215,6 @@ class CommonDatautils { ElasticUtils.executeSqlQuery(sql, { map-> // if(map.address && map.address.contains("五角场")){ index++ - println index rets << ["map_id":map.id,"province":map.pname,"city":map.cityname, "district":map.adname, "name":map.name, "address":map.address, "cat":map.type,"lng":map.location.x, "lat":map.location.y] // } }) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy index 15b8005d6..822eee441 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy @@ -52,11 +52,13 @@ class NewGaoDePoiBatch extends BaseBatch{ def cityList = CommonDatautils.findFullNameCitys().reverse() for(def cityObj : cityList){ -// if(cityObj.city == "上海市"){ +// if(cityObj.city == "胡杨河市"){ def results = CommonDatautils.findAllNewPoiDatas(cityObj.city) - println cityObj.city+" "+results.size() - def rows = running(cityObj.city, results) - save(rows) + println cityObj.province+" " +cityObj.city+" "+results.size() + if(results){ + def rows = running(cityObj.city, results) + save(rows) + } // } } } @@ -91,7 +93,7 @@ class NewGaoDePoiBatch extends BaseBatch{ }); } latch.await() - return rets; + return rets } def getStandardRegion(mzbRegionMap, district){ @@ -130,33 +132,9 @@ class NewGaoDePoiBatch extends BaseBatch{ } saveObj.name = map.name + if(map.cat){ - def arrs = map.cat.split(";|:") - if(arrs.size()>=1){ - saveObj.type = arrs[0] - } - if(arrs.size()>=2){ - saveObj.secondary_type = arrs[1] - }else{ - saveObj.secondary_type = saveObj.type - } - if(arrs.size()>=3){ - def smallstr = arrs[2] - def smallArr = null - if(smallstr.contains("|")){ - smallArr = smallstr.split("|") - } - if(smallArr && smallArr.size()>=1){ - saveObj.small_type = smallArr[0] - }else{ - saveObj.small_type = smallstr - } - }else{ - saveObj.small_type = saveObj.secondary_type - } - - saveObj.cat = saveObj.type+";"+saveObj.secondary_type+";"+saveObj.small_type - + saveObj = catWash(saveObj, map) } def location = [:] location.lon = DataHandleUtils.getDoubleRound(6, map.lng) @@ -176,16 +154,55 @@ class NewGaoDePoiBatch extends BaseBatch{ def towns = townMap.get(saveObj.district) if(location){ - if(towns){ - for(def town : towns){ - if(town.shape && town.shape.coordinates){ - def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates)) - if(flag){ - saveObj.township = town.street - break - } + saveObj = addTown(saveObj, towns, location) + } + + if(map.address){ + saveObj = addressWash(saveObj, map, mzbRegionMap, towns) + } + return saveObj + } + + def catWash(saveObj, map){ + def arrs = map.cat.split(";|:") + if(arrs.size()>=1){ + saveObj.type = arrs[0] + } + if(arrs.size()>=2){ + saveObj.secondary_type = arrs[1] + }else{ + saveObj.secondary_type = saveObj.type + } + if(arrs.size()>=3){ + def smallstr = arrs[2] + def smallArr = null + if(smallstr.contains("|")){ + smallArr = smallstr.split("|") + } + if(smallArr && smallArr.size()>=1){ + saveObj.small_type = smallArr[0] + }else{ + saveObj.small_type = smallstr + } + }else{ + saveObj.small_type = saveObj.secondary_type + } + + saveObj.cat = saveObj.type+";"+saveObj.secondary_type+";"+saveObj.small_type + return saveObj + } + + def addTown(saveObj, towns, location){ + if(towns){ + for(def town : towns){ + if(town.shape && town.shape.coordinates){ + def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates)) + if(flag){ + saveObj.township = town.street + break } } + } // if(map.address && !saveObj.township){ // for(def town : towns){ // if(map.address.contains(town.street)){ @@ -197,86 +214,92 @@ class NewGaoDePoiBatch extends BaseBatch{ // println map.address +" 含街道,镇,乡" // } // } - } } + return saveObj + } - if(map.address){ - if(map.address.length()<3){ - saveObj.address = "" - }else{ - def address = map.address - def i = 0 - while(i<5){ - if(address && address.toString().startsWith(map.province)){ - address = address.toString().substring(map.province.toString().length()) - } - if(address && address.toString().startsWith(map.city)){ - address = address.toString().substring(map.city.toString().length()) - } + def addressWash(saveObj, map, mzbRegionMap, towns){ + if(map.address.length()<3){ + saveObj.address = "" + }else{ + def address = map.address + def i = 0 + while(i<5){ + if(address && address.toString().startsWith(map.province)){ + address = address.toString().substring(map.province.toString().length()) + } + if(address && address.toString().startsWith(map.city)){ + address = address.toString().substring(map.city.toString().length()) + } - if(address && saveObj.district){ - def tjjObj = mzbRegionMap.get(saveObj.district) - if(tjjObj){ - def alias = tjjObj.alias - if(!alias){ - alias = [] + if(address && saveObj.district){ + def tjjObj = mzbRegionMap.get(saveObj.district) + if(tjjObj){ + def alias = null + if(tjjObj.alias){ + alias = new CopyOnWriteArrayList<>(tjjObj.alias); + }else{ + alias = new CopyOnWriteArrayList<>(); + } + + if(!alias.contains(saveObj.district)){ + alias << saveObj.district + } + if(alias.size()>=2){ + alias = alias.sort(){ a, b -> + b.length().compareTo(a.length()); } - if(!alias.contains(saveObj.district)){ - alias << saveObj.district - } -// if(alias.size()>=2){ -// alias.sort(){ a, b -> -// b.length().compareTo(a.length()); -// } -// } + } - for(def regionName : alias){ - if(address.startsWith(regionName) && (regionName.endsWith("区") || regionName.endsWith("县"))){ - address = address.substring(regionName.toString().length()) - break - } + for(def regionName : alias){ + if(address.startsWith(regionName) && (regionName.endsWith("区") || regionName.endsWith("县"))){ + address = address.substring(regionName.toString().length()) + break } } } + } - if(address && address.toString().startsWith("市辖区")){ - address = address.toString().substring(3) - } + if(address && address.toString().startsWith("市辖区")){ + address = address.toString().substring(3) + } - if(address && towns){ - for(def town : towns){ - def townAlias = town.alias - if(!townAlias){ - townAlias = [] + if(address && towns){ + for(def town : towns){ + def townAlias = null + if(town.alias){ + townAlias = new CopyOnWriteArrayList<>(town.alias); + }else{ + townAlias = new CopyOnWriteArrayList<>(); + } + + if(!townAlias.contains(town.street)){ + townAlias << town.street + } + if(townAlias.size()>=2){ + townAlias = townAlias.sort(){ a, b -> + b.length().compareTo(a.length()); + } } - if(!townAlias.contains(town.street)){ - townAlias << town.street - } -// if(townAlias.size()>=2){ -// townAlias = townAlias.sort(){ a, b -> -// b.length().compareTo(a.length()); -// } -// } - for(def townName : townAlias){ - if(address.startsWith(townName)){ - address = address.substring(townName.toString().length()) + for(def townName : townAlias){ + if(address.startsWith(townName)){ + address = address.substring(townName.toString().length()) // if(!saveObj.township){ // saveObj.township = town.street // } - break - } + break } } } - i++ - } - if(address && address.startsWith("(") && address.endsWith(")")){ - saveObj.address = address.substring(1, address.length()-1) - }else{ - saveObj.address = address } + i++ + } + if(address && address.startsWith("(") && address.endsWith(")")){ + saveObj.address = address.substring(1, address.length()-1) + }else{ + saveObj.address = address } } return saveObj @@ -284,7 +307,7 @@ class NewGaoDePoiBatch extends BaseBatch{ def save(rows){ BulkRequest request = new BulkRequest(); - int batchSize = 1000; + int batchSize = 2000; int processedCount = 0; def client = ElasticUtils89.getClient("192.168.0.89")