Merge remote-tracking branch 'origin/master'
This commit is contained in:
+5
-3
@@ -4213,9 +4213,11 @@ class CommonDatautils {
|
||||
def rets = []
|
||||
def index= 0
|
||||
ElasticUtils.executeSqlQuery(sql, { map->
|
||||
index++
|
||||
println index
|
||||
rets << ["map_id":map.id,"province":map.pname,"city":map.cityname, "district":map.adname, "name":map.name, "address":map.address, "cat":map.type,"lng":map.location.x, "lat":map.location.y]
|
||||
// if(map.address && map.address.contains("五角场")){
|
||||
index++
|
||||
println index
|
||||
rets << ["map_id":map.id,"province":map.pname,"city":map.cityname, "district":map.adname, "name":map.name, "address":map.address, "cat":map.type,"lng":map.location.x, "lat":map.location.y]
|
||||
// }
|
||||
})
|
||||
return rets;
|
||||
}
|
||||
|
||||
+104
-30
@@ -29,6 +29,8 @@ class NewGaoDePoiBatch extends BaseBatch{
|
||||
public static def version = new Date().getTime()
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity()
|
||||
|
||||
public static def TAGS = []
|
||||
|
||||
static main(args) {
|
||||
@@ -50,12 +52,12 @@ class NewGaoDePoiBatch extends BaseBatch{
|
||||
|
||||
def cityList = CommonDatautils.findFullNameCitys().reverse()
|
||||
for(def cityObj : cityList){
|
||||
if(cityObj.city == "上海市"){
|
||||
// if(cityObj.city == "上海市"){
|
||||
def results = CommonDatautils.findAllNewPoiDatas(cityObj.city)
|
||||
println cityObj.city+" "+results.size()
|
||||
def rows = running(cityObj.city, results)
|
||||
save(rows)
|
||||
}
|
||||
// }
|
||||
}
|
||||
}
|
||||
def getAllPoiType(filePath){
|
||||
@@ -72,13 +74,14 @@ class NewGaoDePoiBatch extends BaseBatch{
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(results.size())
|
||||
def townMap = CommonDatautils.queryTownByCity(city, ["city","district","mzb_district_name", "street", "shape","alias"])
|
||||
def mzbRegionMap = regionFullNameMap.get(city)
|
||||
for(def i=0;i<results.size();i++){
|
||||
def map = results[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
rets.addAll(createData(map, townMap))
|
||||
rets.addAll(createData(map, mzbRegionMap, townMap))
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
@@ -91,13 +94,37 @@ class NewGaoDePoiBatch extends BaseBatch{
|
||||
return rets;
|
||||
}
|
||||
|
||||
def createData(map, townMap){
|
||||
def getStandardRegion(mzbRegionMap, district){
|
||||
def tjjObj = mzbRegionMap.get(district)
|
||||
if(tjjObj && tjjObj.mzb_district_name && tjjObj.mzb_district_name == district){
|
||||
return district
|
||||
}
|
||||
def updateRegion = null
|
||||
if(tjjObj){
|
||||
if(tjjObj.mzb_district_name && tjjObj.mzb_district_name!=district){
|
||||
updateRegion = tjjObj.mzb_district_name
|
||||
}
|
||||
}else{
|
||||
def set = mzbRegionMap.keySet()
|
||||
for(def key : set){
|
||||
def tempObj = mzbRegionMap.get(key)
|
||||
def alias = tempObj.alias
|
||||
if(alias.contains(district) || district==tempObj.mzb_district_name){
|
||||
updateRegion = key
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return updateRegion
|
||||
}
|
||||
|
||||
def createData(map, mzbRegionMap, townMap){
|
||||
def saveObj = [:]
|
||||
def towns = townMap.get(map.district)
|
||||
saveObj.province = map.province
|
||||
saveObj.city = map.city
|
||||
if(towns){
|
||||
saveObj.district = map.district
|
||||
def newDistrict = getStandardRegion(mzbRegionMap, map.district)
|
||||
if(newDistrict){
|
||||
saveObj.district = newDistrict
|
||||
}else{
|
||||
println map.city +" "+map.district +" 格式化不出区域"
|
||||
}
|
||||
@@ -147,6 +174,7 @@ class NewGaoDePoiBatch extends BaseBatch{
|
||||
saveObj.status = 1
|
||||
}
|
||||
|
||||
def towns = townMap.get(saveObj.district)
|
||||
if(location){
|
||||
if(towns){
|
||||
for(def town : towns){
|
||||
@@ -158,17 +186,17 @@ class NewGaoDePoiBatch extends BaseBatch{
|
||||
}
|
||||
}
|
||||
}
|
||||
if(map.address && !saveObj.township){
|
||||
for(def town : towns){
|
||||
if(map.address.contains(town.street)){
|
||||
saveObj.township = town.street
|
||||
break
|
||||
}
|
||||
}
|
||||
if(!saveObj.township && map.address && (map.address.contains("街道") || map.address.contains("镇") || map.address.contains("乡"))){
|
||||
println map.address +" 含街道,镇,乡"
|
||||
}
|
||||
}
|
||||
// if(map.address && !saveObj.township){
|
||||
// for(def town : towns){
|
||||
// if(map.address.contains(town.street)){
|
||||
// saveObj.township = town.street
|
||||
// break
|
||||
// }
|
||||
// }
|
||||
// if(!saveObj.township && map.address && (map.address.contains("街道") || map.address.contains("镇") || map.address.contains("乡"))){
|
||||
// println map.address +" 含街道,镇,乡"
|
||||
// }
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -179,20 +207,68 @@ class NewGaoDePoiBatch extends BaseBatch{
|
||||
def address = map.address
|
||||
def i = 0
|
||||
while(i<5){
|
||||
if(address.toString().startsWith(map.province)){
|
||||
address = address.toString().substring(0, map.province.toString().length())
|
||||
if(address && address.toString().startsWith(map.province)){
|
||||
address = address.toString().substring(map.province.toString().length())
|
||||
}
|
||||
if(address.toString().startsWith(map.city)){
|
||||
address = address.toString().substring(0, map.city.toString().length())
|
||||
if(address && address.toString().startsWith(map.city)){
|
||||
address = address.toString().substring(map.city.toString().length())
|
||||
}
|
||||
if(map.district && address.toString().startsWith(map.district)){
|
||||
address = address.toString().substring(0, map.district.toString().length())
|
||||
|
||||
if(address && saveObj.district){
|
||||
def tjjObj = mzbRegionMap.get(saveObj.district)
|
||||
if(tjjObj){
|
||||
def alias = tjjObj.alias
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(!alias.contains(saveObj.district)){
|
||||
alias << saveObj.district
|
||||
}
|
||||
// if(alias.size()>=2){
|
||||
// alias.sort(){ a, b ->
|
||||
// b.length().compareTo(a.length());
|
||||
// }
|
||||
// }
|
||||
|
||||
for(def regionName : alias){
|
||||
if(address.startsWith(regionName) && (regionName.endsWith("区") || regionName.endsWith("县"))){
|
||||
address = address.substring(regionName.toString().length())
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if(address.toString().startsWith("市辖区")){
|
||||
address = address.toString().substring(0, 3)
|
||||
|
||||
|
||||
if(address && address.toString().startsWith("市辖区")){
|
||||
address = address.toString().substring(3)
|
||||
}
|
||||
if(saveObj.township && address.toString().startsWith(saveObj.township)){
|
||||
address = address.toString().substring(0, saveObj.township.toString().length())
|
||||
|
||||
if(address && towns){
|
||||
for(def town : towns){
|
||||
def townAlias = town.alias
|
||||
if(!townAlias){
|
||||
townAlias = []
|
||||
}
|
||||
if(!townAlias.contains(town.street)){
|
||||
townAlias << town.street
|
||||
}
|
||||
// if(townAlias.size()>=2){
|
||||
// townAlias = townAlias.sort(){ a, b ->
|
||||
// b.length().compareTo(a.length());
|
||||
// }
|
||||
// }
|
||||
|
||||
for(def townName : townAlias){
|
||||
if(address.startsWith(townName)){
|
||||
address = address.substring(townName.toString().length())
|
||||
// if(!saveObj.township){
|
||||
// saveObj.township = town.street
|
||||
// }
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
i++
|
||||
}
|
||||
@@ -201,8 +277,6 @@ class NewGaoDePoiBatch extends BaseBatch{
|
||||
}else{
|
||||
saveObj.address = address
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
return saveObj
|
||||
|
||||
Reference in New Issue
Block a user