乡镇街道标准化

This commit is contained in:
2024-11-01 11:01:55 +08:00
parent 199d0ea71e
commit b3a1114cc4
@@ -0,0 +1,251 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.mongodb.BasicDBObject
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.update.UpdateRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.common.xcontent.XContentType
/**
* 区域标准化
* 1.小区库乡镇街道标准化,包含小区,商铺,写字楼
* 2.poi数据库中乡镇街道标准化
*/
class TownStandardBatch extends BaseBatch{
def fullCityMap = CommonDatautils.queryFullNameCity()
def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity()
def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity()
static main(args) {
runBatch(new TownStandardBatch(), args)
}
@Override
void process(Object args) {
TownStandard()
}
def TownStandard(){
townStandardByCat("office")
// townStandardByPoi()
}
def townStandardByCat(def cat){
//"三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德",
def citys = CommonDatautils.findAllCity()
for(def city : citys){
def data = fullCityMap.get(city)
def fullCityName = null
if(data){
fullCityName = data.city
}
def mzbRegionMap = CommonDatautils.queryMzbRegion(fullCityName)
def shortRegionNameMap = CommonDatautils.getRegionShortName(fullCityName)
def townMap = CommonDatautils.queryTownByCity(fullCityName, ["city","district", "street", "alias"], mzbRegionMap)
if(townMap){
def regionSet = townMap.keySet()
if(regionSet.size()>1){
haveRegionProcess(city, fullCityName, townMap, cat, shortRegionNameMap)
}else{
println city +" 无区域"
}
}
}
}
def townStandardByPoi(){
def citys = ["重庆"]//CommonDatautils.findAllCity()
for(def city : citys){
def data = fullCityMap.get(city)
def fullCityName = null
if(data){
fullCityName = data.city
}
if(fullCityName){
def rMap = regionFullNameMap.get(fullCityName)
if(rMap){
def set = rMap.keySet()
def rets = findEsPoiByCity(fullCityName)
println city +" "+rets.size()
if(set.size()>1){
poiHaveRegionProcess(city, set, rMap, rets)
}else{
// haveStreetProcess(city, set, cat)
}
}
}else{
println "没找到相应的全称:"+city
}
}
}
def haveRegionProcess(city, fullCityName, townMap, cat, shortRegionNameMap){
def results = []
if(fullCityName){
def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name", "township"])
for(def d : districts){
if(d.township){
def towns = null
def updateTown = null
for(def m : townMap){
def fullRegionName = shortRegionNameMap.get(d.region)
if(m.key == fullRegionName){
towns = m.value
break
}
}
def opt = true
if(towns){
for(def town : towns){
def townAlias = []
if(town.alias){
townAlias = town.alias
}
townAlias << town.street
if(townAlias.contains(d.township)){
if(d.township!=town.street){
updateTown = town.street
}else{
opt = false
}
break
}
}
}
if(opt){
if(updateTown){
def udata = new BasicDBObject()
udata.put("township", updateTown)
udata.put("m_date", new Date())
println d.city +" "+d.region+" "+d.name+" "+d.township +" 乡镇修改为:"+updateTown
// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
}else{
def udata = new BasicDBObject()
udata.put("township", "")
udata.put("m_date", new Date())
println d.city +" "+d.region+" "+d.name+" "+d.township +" 乡镇修改为:空值"
// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
}
}
}
}
}
}
def updateRegionByEs(rows){
def client = ElasticUtils89.getClient("192.168.0.89")
def bulkRequest = new BulkRequest()
rows.each { row ->
def id = row.keySet().iterator().next()
def doc = row.get(id)
def updateRequest = new UpdateRequest("didata_map_poi", id)
updateRequest.doc(doc, XContentType.JSON)
bulkRequest.add(updateRequest)
}
try {
// 执行批量更新请求
def bulkResponse = client.bulk(bulkRequest, RequestOptions.DEFAULT)
// 检查响应
if (bulkResponse.hasFailures()) {
println("批量更新中有失败的操作:")
bulkResponse.buildFailureMessage().each { println(it) }
} else {
println("批量更新成功!")
}
} catch (Exception e) {
e.printStackTrace()
} finally {
// 关闭客户端
try {
client.close()
} catch (Exception e) {
e.printStackTrace()
}
}
}
// def queryPoiDatasByCity(){
// def esUrl = "http://localhost:9200/your_index/_search"
// def pageSize = 1000 // 每页返回的记录数
// def scrollSize = '1m' // scroll上下文的生存时间
//
//// 初始化HttpClient和ObjectMapper
// CloseableHttpClient httpClient = HttpClients.createDefault()
// ObjectMapper objectMapper = new ObjectMapper()
//
//// 初始化scroll请求
// def scrollRequest = """
//{
// "size": $pageSize,
// "query": {
// "match_all": {}
// },
// "scroll": "$scrollSize"
//}
//"""
//
//// 发送scroll初始化请求
// HttpGet scrollInitRequest = new HttpGet(esUrl)
// scrollInitRequest.setEntity(new org.apache.http.entity.StringEntity(scrollRequest, "UTF-8"))
// scrollInitRequest.setHeader("Content-Type", "application/json")
//
// HttpResponse scrollInitResponse = httpClient.execute(scrollInitRequest)
// HttpEntity entity = scrollInitResponse.getEntity()
// String scrollId = objectMapper.readValue(EntityUtils.toString(entity), Map.class).get("_scroll_id").toString()
//
//// 存储所有结果
// def allResults = []
//
//// 发送scroll请求获取数据,直到没有更多数据为止
// while (true) {
// def scrollRequestWithId = """
// {
// "scroll_id": "$scrollId",
// "scroll": "$scrollSize"
// }
// """
//
// HttpGet scrollGetRequest = new HttpGet(esUrl + "?scroll")
// scrollGetRequest.setEntity(new org.apache.http.entity.StringEntity(scrollRequestWithId, "UTF-8"))
// scrollGetRequest.setHeader("Content-Type", "application/json")
//
// HttpResponse scrollGetResponse = httpClient.execute(scrollGetRequest)
// entity = scrollGetResponse.getEntity()
// def responseMap = objectMapper.readValue(EntityUtils.toString(entity), Map.class)
//
// def hits = responseMap.get("hits").get("hits")
// if (hits.isEmpty()) {
// break // 没有更多数据,跳出循环
// }
//
// // 合并当前页的数据
// allResults.addAll(hits.map({ it.get("_source") }))
//
// // 更新scrollId以进行下一次请求
// scrollId = responseMap.get("_scroll_id").toString()
// }
//
// // 关闭HttpClient
// httpClient.close()
//
// // 输出或处理全量数据
// println(allResults)
//
// // 注意:实际应用中,你可能需要在获取完数据后清除scroll上下文,以释放资源。
// }
}