-1
@@ -4215,7 +4215,6 @@ class CommonDatautils {
|
||||
ElasticUtils.executeSqlQuery(sql, { map->
|
||||
// if(map.address && map.address.contains("五角场")){
|
||||
index++
|
||||
println index
|
||||
rets << ["map_id":map.id,"province":map.pname,"city":map.cityname, "district":map.adname, "name":map.name, "address":map.address, "cat":map.type,"lng":map.location.x, "lat":map.location.y]
|
||||
// }
|
||||
})
|
||||
|
||||
+123
-100
@@ -52,11 +52,13 @@ class NewGaoDePoiBatch extends BaseBatch{
|
||||
|
||||
def cityList = CommonDatautils.findFullNameCitys().reverse()
|
||||
for(def cityObj : cityList){
|
||||
// if(cityObj.city == "上海市"){
|
||||
// if(cityObj.city == "胡杨河市"){
|
||||
def results = CommonDatautils.findAllNewPoiDatas(cityObj.city)
|
||||
println cityObj.city+" "+results.size()
|
||||
def rows = running(cityObj.city, results)
|
||||
save(rows)
|
||||
println cityObj.province+" " +cityObj.city+" "+results.size()
|
||||
if(results){
|
||||
def rows = running(cityObj.city, results)
|
||||
save(rows)
|
||||
}
|
||||
// }
|
||||
}
|
||||
}
|
||||
@@ -91,7 +93,7 @@ class NewGaoDePoiBatch extends BaseBatch{
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets;
|
||||
return rets
|
||||
}
|
||||
|
||||
def getStandardRegion(mzbRegionMap, district){
|
||||
@@ -130,33 +132,9 @@ class NewGaoDePoiBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
saveObj.name = map.name
|
||||
|
||||
if(map.cat){
|
||||
def arrs = map.cat.split(";|:")
|
||||
if(arrs.size()>=1){
|
||||
saveObj.type = arrs[0]
|
||||
}
|
||||
if(arrs.size()>=2){
|
||||
saveObj.secondary_type = arrs[1]
|
||||
}else{
|
||||
saveObj.secondary_type = saveObj.type
|
||||
}
|
||||
if(arrs.size()>=3){
|
||||
def smallstr = arrs[2]
|
||||
def smallArr = null
|
||||
if(smallstr.contains("|")){
|
||||
smallArr = smallstr.split("|")
|
||||
}
|
||||
if(smallArr && smallArr.size()>=1){
|
||||
saveObj.small_type = smallArr[0]
|
||||
}else{
|
||||
saveObj.small_type = smallstr
|
||||
}
|
||||
}else{
|
||||
saveObj.small_type = saveObj.secondary_type
|
||||
}
|
||||
|
||||
saveObj.cat = saveObj.type+";"+saveObj.secondary_type+";"+saveObj.small_type
|
||||
|
||||
saveObj = catWash(saveObj, map)
|
||||
}
|
||||
def location = [:]
|
||||
location.lon = DataHandleUtils.getDoubleRound(6, map.lng)
|
||||
@@ -176,16 +154,55 @@ class NewGaoDePoiBatch extends BaseBatch{
|
||||
|
||||
def towns = townMap.get(saveObj.district)
|
||||
if(location){
|
||||
if(towns){
|
||||
for(def town : towns){
|
||||
if(town.shape && town.shape.coordinates){
|
||||
def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates))
|
||||
if(flag){
|
||||
saveObj.township = town.street
|
||||
break
|
||||
}
|
||||
saveObj = addTown(saveObj, towns, location)
|
||||
}
|
||||
|
||||
if(map.address){
|
||||
saveObj = addressWash(saveObj, map, mzbRegionMap, towns)
|
||||
}
|
||||
return saveObj
|
||||
}
|
||||
|
||||
def catWash(saveObj, map){
|
||||
def arrs = map.cat.split(";|:")
|
||||
if(arrs.size()>=1){
|
||||
saveObj.type = arrs[0]
|
||||
}
|
||||
if(arrs.size()>=2){
|
||||
saveObj.secondary_type = arrs[1]
|
||||
}else{
|
||||
saveObj.secondary_type = saveObj.type
|
||||
}
|
||||
if(arrs.size()>=3){
|
||||
def smallstr = arrs[2]
|
||||
def smallArr = null
|
||||
if(smallstr.contains("|")){
|
||||
smallArr = smallstr.split("|")
|
||||
}
|
||||
if(smallArr && smallArr.size()>=1){
|
||||
saveObj.small_type = smallArr[0]
|
||||
}else{
|
||||
saveObj.small_type = smallstr
|
||||
}
|
||||
}else{
|
||||
saveObj.small_type = saveObj.secondary_type
|
||||
}
|
||||
|
||||
saveObj.cat = saveObj.type+";"+saveObj.secondary_type+";"+saveObj.small_type
|
||||
return saveObj
|
||||
}
|
||||
|
||||
def addTown(saveObj, towns, location){
|
||||
if(towns){
|
||||
for(def town : towns){
|
||||
if(town.shape && town.shape.coordinates){
|
||||
def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates))
|
||||
if(flag){
|
||||
saveObj.township = town.street
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
// if(map.address && !saveObj.township){
|
||||
// for(def town : towns){
|
||||
// if(map.address.contains(town.street)){
|
||||
@@ -197,86 +214,92 @@ class NewGaoDePoiBatch extends BaseBatch{
|
||||
// println map.address +" 含街道,镇,乡"
|
||||
// }
|
||||
// }
|
||||
}
|
||||
}
|
||||
return saveObj
|
||||
}
|
||||
|
||||
if(map.address){
|
||||
if(map.address.length()<3){
|
||||
saveObj.address = ""
|
||||
}else{
|
||||
def address = map.address
|
||||
def i = 0
|
||||
while(i<5){
|
||||
if(address && address.toString().startsWith(map.province)){
|
||||
address = address.toString().substring(map.province.toString().length())
|
||||
}
|
||||
if(address && address.toString().startsWith(map.city)){
|
||||
address = address.toString().substring(map.city.toString().length())
|
||||
}
|
||||
def addressWash(saveObj, map, mzbRegionMap, towns){
|
||||
if(map.address.length()<3){
|
||||
saveObj.address = ""
|
||||
}else{
|
||||
def address = map.address
|
||||
def i = 0
|
||||
while(i<5){
|
||||
if(address && address.toString().startsWith(map.province)){
|
||||
address = address.toString().substring(map.province.toString().length())
|
||||
}
|
||||
if(address && address.toString().startsWith(map.city)){
|
||||
address = address.toString().substring(map.city.toString().length())
|
||||
}
|
||||
|
||||
if(address && saveObj.district){
|
||||
def tjjObj = mzbRegionMap.get(saveObj.district)
|
||||
if(tjjObj){
|
||||
def alias = tjjObj.alias
|
||||
if(!alias){
|
||||
alias = []
|
||||
if(address && saveObj.district){
|
||||
def tjjObj = mzbRegionMap.get(saveObj.district)
|
||||
if(tjjObj){
|
||||
def alias = null
|
||||
if(tjjObj.alias){
|
||||
alias = new CopyOnWriteArrayList<>(tjjObj.alias);
|
||||
}else{
|
||||
alias = new CopyOnWriteArrayList<>();
|
||||
}
|
||||
|
||||
if(!alias.contains(saveObj.district)){
|
||||
alias << saveObj.district
|
||||
}
|
||||
if(alias.size()>=2){
|
||||
alias = alias.sort(){ a, b ->
|
||||
b.length().compareTo(a.length());
|
||||
}
|
||||
if(!alias.contains(saveObj.district)){
|
||||
alias << saveObj.district
|
||||
}
|
||||
// if(alias.size()>=2){
|
||||
// alias.sort(){ a, b ->
|
||||
// b.length().compareTo(a.length());
|
||||
// }
|
||||
// }
|
||||
}
|
||||
|
||||
for(def regionName : alias){
|
||||
if(address.startsWith(regionName) && (regionName.endsWith("区") || regionName.endsWith("县"))){
|
||||
address = address.substring(regionName.toString().length())
|
||||
break
|
||||
}
|
||||
for(def regionName : alias){
|
||||
if(address.startsWith(regionName) && (regionName.endsWith("区") || regionName.endsWith("县"))){
|
||||
address = address.substring(regionName.toString().length())
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if(address && address.toString().startsWith("市辖区")){
|
||||
address = address.toString().substring(3)
|
||||
}
|
||||
if(address && address.toString().startsWith("市辖区")){
|
||||
address = address.toString().substring(3)
|
||||
}
|
||||
|
||||
if(address && towns){
|
||||
for(def town : towns){
|
||||
def townAlias = town.alias
|
||||
if(!townAlias){
|
||||
townAlias = []
|
||||
if(address && towns){
|
||||
for(def town : towns){
|
||||
def townAlias = null
|
||||
if(town.alias){
|
||||
townAlias = new CopyOnWriteArrayList<>(town.alias);
|
||||
}else{
|
||||
townAlias = new CopyOnWriteArrayList<>();
|
||||
}
|
||||
|
||||
if(!townAlias.contains(town.street)){
|
||||
townAlias << town.street
|
||||
}
|
||||
if(townAlias.size()>=2){
|
||||
townAlias = townAlias.sort(){ a, b ->
|
||||
b.length().compareTo(a.length());
|
||||
}
|
||||
}
|
||||
if(!townAlias.contains(town.street)){
|
||||
townAlias << town.street
|
||||
}
|
||||
// if(townAlias.size()>=2){
|
||||
// townAlias = townAlias.sort(){ a, b ->
|
||||
// b.length().compareTo(a.length());
|
||||
// }
|
||||
// }
|
||||
|
||||
for(def townName : townAlias){
|
||||
if(address.startsWith(townName)){
|
||||
address = address.substring(townName.toString().length())
|
||||
for(def townName : townAlias){
|
||||
if(address.startsWith(townName)){
|
||||
address = address.substring(townName.toString().length())
|
||||
// if(!saveObj.township){
|
||||
// saveObj.township = town.street
|
||||
// }
|
||||
break
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
i++
|
||||
}
|
||||
if(address && address.startsWith("(") && address.endsWith(")")){
|
||||
saveObj.address = address.substring(1, address.length()-1)
|
||||
}else{
|
||||
saveObj.address = address
|
||||
}
|
||||
i++
|
||||
}
|
||||
if(address && address.startsWith("(") && address.endsWith(")")){
|
||||
saveObj.address = address.substring(1, address.length()-1)
|
||||
}else{
|
||||
saveObj.address = address
|
||||
}
|
||||
}
|
||||
return saveObj
|
||||
@@ -284,7 +307,7 @@ class NewGaoDePoiBatch extends BaseBatch{
|
||||
|
||||
def save(rows){
|
||||
BulkRequest request = new BulkRequest();
|
||||
int batchSize = 1000;
|
||||
int batchSize = 2000;
|
||||
int processedCount = 0;
|
||||
def client = ElasticUtils89.getClient("192.168.0.89")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user