乡镇街道标准化修改
This commit is contained in:
+11
-6
@@ -4858,6 +4858,7 @@ class CommonDatautils {
|
||||
keys.put("city",1)
|
||||
keys.put("district",1)
|
||||
keys.put("alias",1)
|
||||
keys.put("mzb_district_name",1)
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def retMap = [:]
|
||||
for(def d : dbcursor){
|
||||
@@ -4866,7 +4867,7 @@ class CommonDatautils {
|
||||
regionMap = [:]
|
||||
retMap.put(d.city, regionMap)
|
||||
}
|
||||
regionMap.put(d.district, d.alias)
|
||||
regionMap.put(d.district, d)
|
||||
}
|
||||
|
||||
return retMap
|
||||
@@ -4975,18 +4976,22 @@ class CommonDatautils {
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def townMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def list = townMap.get(d.district)
|
||||
if(!list){
|
||||
list = []
|
||||
if(d.mzb_district_name){
|
||||
def list = townMap.get(d.mzb_district_name)
|
||||
if(!list){
|
||||
list = []
|
||||
}
|
||||
list << d
|
||||
townMap.put(d.mzb_district_name, list)
|
||||
}
|
||||
list << d
|
||||
townMap.put(d.district, list)
|
||||
}
|
||||
|
||||
println "[${city}] 乡镇街道 加载完成 "+new Date().format("yyyy-MM-dd HH:mm:ss")
|
||||
return townMap
|
||||
}
|
||||
|
||||
|
||||
|
||||
static def queryTownByCity(city){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
|
||||
+50
-30
@@ -23,6 +23,7 @@ class RegionStandardBatch extends BaseBatch{
|
||||
def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity()
|
||||
def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity()
|
||||
|
||||
|
||||
static main(args) {
|
||||
runBatch(new RegionStandardBatch(), args)
|
||||
}
|
||||
@@ -37,14 +38,15 @@ class RegionStandardBatch extends BaseBatch{
|
||||
seaweedRegionStandardByCat("district")
|
||||
}
|
||||
def poiRegionStandard(){
|
||||
|
||||
def runCitys = ["重庆","三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德"]
|
||||
def index = 0
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def citys = ["攀枝花"]//CommonDatautils.findAllCity()
|
||||
for(def city : citys){
|
||||
index ++
|
||||
if(index<32){
|
||||
continue
|
||||
}
|
||||
// if(index<32){
|
||||
// continue
|
||||
// }
|
||||
if(runCitys.contains(city)){
|
||||
continue
|
||||
}
|
||||
@@ -74,6 +76,7 @@ class RegionStandardBatch extends BaseBatch{
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -195,39 +198,56 @@ class RegionStandardBatch extends BaseBatch{
|
||||
def poiHaveRegionProcess(city, set, rMap, datas){
|
||||
def results = []
|
||||
for(def d : datas){
|
||||
if(set.contains(d.district)){
|
||||
def tjjObj = rMap.get(d.district)
|
||||
if(tjjObj && tjjObj.mzb_district_name && tjjObj.mzb_district_name==d.district){
|
||||
continue
|
||||
}
|
||||
def updateRegion = null
|
||||
for(def key : set){
|
||||
def alias = rMap.get(key)
|
||||
if(alias.contains(d.district)){
|
||||
updateRegion = key
|
||||
break;
|
||||
def updateFlag = true
|
||||
if(tjjObj){
|
||||
if(tjjObj.mzb_district_name && tjjObj.mzb_district_name!=d.district){
|
||||
updateRegion = tjjObj.mzb_district_name
|
||||
}
|
||||
}
|
||||
if(updateRegion){
|
||||
def m = [:]
|
||||
m.put(d.id, ["district":updateRegion])
|
||||
results << m
|
||||
if(results.size()%100==0){
|
||||
updateRegionByEs(results)
|
||||
results.clear()
|
||||
}
|
||||
println d.city +" "+d.district+" "+d.name +" "+d.status+" 区域修改为:"+updateRegion
|
||||
|
||||
}else{
|
||||
def m = [:]
|
||||
m.put(d.id, ["status":1])
|
||||
results << m
|
||||
if(results.size()%100==0){
|
||||
updateRegionByEs(results)
|
||||
results.clear()
|
||||
for(def key : set){
|
||||
def tempObj = rMap.get(key)
|
||||
def alias = tempObj.alias
|
||||
if(alias.contains(d.district) || d.district==tempObj.mzb_district_name){
|
||||
if(d.district!=tempObj.mzb_district_name){
|
||||
updateRegion = key
|
||||
}else{
|
||||
updateFlag = false
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
println d.city +" "+d.district+" "+d.name +" "+d.status+" 状态修改为:删除"
|
||||
|
||||
}
|
||||
|
||||
if(updateFlag){
|
||||
if(updateRegion){
|
||||
def m = [:]
|
||||
m.put(d.id, ["district":updateRegion])
|
||||
results << m
|
||||
if(results.size()%100==0){
|
||||
updateRegionByEs(results)
|
||||
results.clear()
|
||||
}
|
||||
|
||||
println d.city +" "+d.district+" "+d.name +" "+d.status+" 区域修改为:"+updateRegion
|
||||
|
||||
}else{
|
||||
def m = [:]
|
||||
m.put(d.id, ["status":1])
|
||||
results << m
|
||||
if(results.size()%100==0){
|
||||
updateRegionByEs(results)
|
||||
results.clear()
|
||||
}
|
||||
|
||||
println d.city +" "+d.district+" "+d.name +" "+d.status+" 状态修改为:删除"
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(results.size()>0){
|
||||
|
||||
+63
-23
@@ -1,17 +1,17 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.apache.commons.io.FileUtils
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.update.UpdateRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.common.xcontent.XContentType
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
/**
|
||||
* 区域标准化
|
||||
* 1.小区库乡镇街道标准化,包含小区,商铺,写字楼
|
||||
@@ -23,12 +23,7 @@ class TownStandardBatch extends BaseBatch{
|
||||
def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity()
|
||||
def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity()
|
||||
|
||||
def poiEsUrl = "http://192.168.0.89:9200/didata_map_poi/_search"
|
||||
|
||||
def elasticsearchUrl = "http://192.168.0.89:9200/_search/scroll"
|
||||
def initialSearchUrl = "http://192.168.0.89:9200/didata_map_poi/_search?scroll=1m"
|
||||
def scrollTime = "1m" // Scroll context time-to-live
|
||||
def pageSize = 1000 // Number of hits to return per scroll
|
||||
def updateList = []
|
||||
|
||||
static main(args) {
|
||||
runBatch(new TownStandardBatch(), args)
|
||||
@@ -70,7 +65,8 @@ class TownStandardBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
def townStandardByPoi(){
|
||||
def citys = ["三明"]//CommonDatautils.findAllCity()
|
||||
updateList << ["数据ID", "城市","区域","乡镇街道","名称","状态", "修改"]
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
for(def city : citys){
|
||||
|
||||
def data = fullCityMap.get(city)
|
||||
@@ -79,22 +75,33 @@ class TownStandardBatch extends BaseBatch{
|
||||
fullCityName = data.city
|
||||
}
|
||||
if(fullCityName){
|
||||
def townMap = CommonDatautils.queryTownByCity(fullCityName, ["city","district", "street", "alias"])
|
||||
def rMap = regionFullNameMap.get(fullCityName)
|
||||
if(rMap){
|
||||
def set = rMap.keySet()
|
||||
def rets = findEsPoiByCity(fullCityName)
|
||||
println city +" "+rets.size()
|
||||
if(set.size()>1){
|
||||
poiHaveRegionProcess(fullCityName, townMap, rets)
|
||||
}else{
|
||||
|
||||
def fileName = "/home/goojia/gj/poi/${fullCityName}.csv"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\${fullCityName}.csv"
|
||||
}
|
||||
File file=new File(fileName);
|
||||
if(file.exists()){
|
||||
def rMap = regionFullNameMap.get(fullCityName)
|
||||
if(rMap){
|
||||
def mzbRegionMap = CommonDatautils.queryMzbRegion(fullCityName)
|
||||
def townMap = CommonDatautils.queryTownByCity(fullCityName, ["city","district","mzb_district_name", "street", "alias"])
|
||||
def set = rMap.keySet()
|
||||
// def rets = findEsPoiByCity(fullCityName)
|
||||
def rets = readPoiCsv(fullCityName)
|
||||
println city +" "+rets.size()
|
||||
if(set.size()>1){
|
||||
poiHaveRegionProcess(fullCityName, townMap, rets, mzbRegionMap)
|
||||
}else{
|
||||
// haveStreetProcess(city, set, cat)
|
||||
}
|
||||
}
|
||||
}
|
||||
}else{
|
||||
println "没找到相应的全称:"+city
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\乡镇街道待查数据.xlsx", updateList, "数据")
|
||||
}
|
||||
def haveRegionProcess(city, fullCityName, townMap, cat, shortRegionNameMap){
|
||||
def results = []
|
||||
@@ -149,12 +156,15 @@ class TownStandardBatch extends BaseBatch{
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
def poiHaveRegionProcess(fullCityName, townMap, rets){
|
||||
def poiHaveRegionProcess(fullCityName, townMap, rets, mzbRegionMap){
|
||||
def results = []
|
||||
if(fullCityName){
|
||||
for(def d : rets){
|
||||
|
||||
if(d.township){
|
||||
def towns = null
|
||||
def updateTown = null
|
||||
@@ -163,6 +173,8 @@ class TownStandardBatch extends BaseBatch{
|
||||
if(townDatas){
|
||||
towns = townDatas
|
||||
break
|
||||
}else{
|
||||
println(d.city +" "+d.district+" 没找下辖街道")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -193,6 +205,7 @@ class TownStandardBatch extends BaseBatch{
|
||||
// updateByEs(results)
|
||||
// results.clear()
|
||||
// }
|
||||
updateList << [d.id, d.city, d.district, d.name,d.township, d.status, updateTown]
|
||||
println d.city +" "+d.district+" "+d.name +" "+d.township+" 乡镇修改为:"+updateTown
|
||||
}else{
|
||||
def m = [:]
|
||||
@@ -202,6 +215,7 @@ class TownStandardBatch extends BaseBatch{
|
||||
// updateByEs(results)
|
||||
// results.clear()
|
||||
// }
|
||||
updateList << [d.id, d.city, d.district, d.name, d.township, d.status, ""]
|
||||
println d.city +" "+d.district+" "+d.name +" "+d.township+" 状态修改为:空值"
|
||||
}
|
||||
}
|
||||
@@ -210,10 +224,10 @@ class TownStandardBatch extends BaseBatch{
|
||||
}
|
||||
}
|
||||
|
||||
if(results.size()>0){
|
||||
updateByEs(results)
|
||||
results.clear()
|
||||
}
|
||||
// if(results.size()>0){
|
||||
// updateByEs(results)
|
||||
// results.clear()
|
||||
// }
|
||||
println fullCityName +" 处理完成"
|
||||
}
|
||||
|
||||
@@ -276,4 +290,30 @@ class TownStandardBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
|
||||
def readPoiCsv(filename){
|
||||
def datas = readCSV(filename)
|
||||
return datas
|
||||
}
|
||||
|
||||
def readCSV(filename){
|
||||
def lines = FileUtils.readLines(new File(filename))
|
||||
def rets = [];
|
||||
for(line in lines) {
|
||||
if(line){
|
||||
String[] arrs = line.split("\\|@\\|")
|
||||
def o = new BasicDBObject()
|
||||
o.put("city", arrs[0])
|
||||
o.put("district", arrs[1])
|
||||
o.put("name", arrs[2])
|
||||
o.put("township", arrs[3])
|
||||
o.put("status", arrs[4])
|
||||
o.put("id", arrs[5])
|
||||
rets << o
|
||||
}
|
||||
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user