+251
@@ -0,0 +1,251 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.update.UpdateRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.common.xcontent.XContentType
|
||||
|
||||
/**
|
||||
* 区域标准化
|
||||
* 1.小区库乡镇街道标准化,包含小区,商铺,写字楼
|
||||
* 2.poi数据库中乡镇街道标准化
|
||||
*/
|
||||
class TownStandardBatch extends BaseBatch{
|
||||
|
||||
def fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity()
|
||||
def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity()
|
||||
|
||||
static main(args) {
|
||||
runBatch(new TownStandardBatch(), args)
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
TownStandard()
|
||||
}
|
||||
|
||||
def TownStandard(){
|
||||
townStandardByCat("office")
|
||||
// townStandardByPoi()
|
||||
}
|
||||
def townStandardByCat(def cat){
|
||||
//"三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德",
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
for(def city : citys){
|
||||
|
||||
def data = fullCityMap.get(city)
|
||||
def fullCityName = null
|
||||
if(data){
|
||||
fullCityName = data.city
|
||||
}
|
||||
|
||||
def mzbRegionMap = CommonDatautils.queryMzbRegion(fullCityName)
|
||||
def shortRegionNameMap = CommonDatautils.getRegionShortName(fullCityName)
|
||||
def townMap = CommonDatautils.queryTownByCity(fullCityName, ["city","district", "street", "alias"], mzbRegionMap)
|
||||
if(townMap){
|
||||
def regionSet = townMap.keySet()
|
||||
if(regionSet.size()>1){
|
||||
haveRegionProcess(city, fullCityName, townMap, cat, shortRegionNameMap)
|
||||
}else{
|
||||
println city +" 无区域"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def townStandardByPoi(){
|
||||
def citys = ["重庆"]//CommonDatautils.findAllCity()
|
||||
for(def city : citys){
|
||||
|
||||
def data = fullCityMap.get(city)
|
||||
def fullCityName = null
|
||||
if(data){
|
||||
fullCityName = data.city
|
||||
}
|
||||
if(fullCityName){
|
||||
|
||||
def rMap = regionFullNameMap.get(fullCityName)
|
||||
if(rMap){
|
||||
def set = rMap.keySet()
|
||||
def rets = findEsPoiByCity(fullCityName)
|
||||
println city +" "+rets.size()
|
||||
if(set.size()>1){
|
||||
poiHaveRegionProcess(city, set, rMap, rets)
|
||||
}else{
|
||||
// haveStreetProcess(city, set, cat)
|
||||
}
|
||||
}
|
||||
}else{
|
||||
println "没找到相应的全称:"+city
|
||||
}
|
||||
}
|
||||
}
|
||||
def haveRegionProcess(city, fullCityName, townMap, cat, shortRegionNameMap){
|
||||
def results = []
|
||||
if(fullCityName){
|
||||
def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name", "township"])
|
||||
for(def d : districts){
|
||||
if(d.township){
|
||||
def towns = null
|
||||
def updateTown = null
|
||||
for(def m : townMap){
|
||||
def fullRegionName = shortRegionNameMap.get(d.region)
|
||||
if(m.key == fullRegionName){
|
||||
towns = m.value
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
def opt = true
|
||||
if(towns){
|
||||
for(def town : towns){
|
||||
def townAlias = []
|
||||
if(town.alias){
|
||||
townAlias = town.alias
|
||||
}
|
||||
townAlias << town.street
|
||||
if(townAlias.contains(d.township)){
|
||||
if(d.township!=town.street){
|
||||
updateTown = town.street
|
||||
}else{
|
||||
opt = false
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if(opt){
|
||||
if(updateTown){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("township", updateTown)
|
||||
udata.put("m_date", new Date())
|
||||
println d.city +" "+d.region+" "+d.name+" "+d.township +" 乡镇修改为:"+updateTown
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
|
||||
}else{
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("township", "")
|
||||
udata.put("m_date", new Date())
|
||||
println d.city +" "+d.region+" "+d.name+" "+d.township +" 乡镇修改为:空值"
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def updateRegionByEs(rows){
|
||||
def client = ElasticUtils89.getClient("192.168.0.89")
|
||||
def bulkRequest = new BulkRequest()
|
||||
|
||||
rows.each { row ->
|
||||
def id = row.keySet().iterator().next()
|
||||
def doc = row.get(id)
|
||||
|
||||
def updateRequest = new UpdateRequest("didata_map_poi", id)
|
||||
updateRequest.doc(doc, XContentType.JSON)
|
||||
bulkRequest.add(updateRequest)
|
||||
}
|
||||
|
||||
try {
|
||||
// 执行批量更新请求
|
||||
def bulkResponse = client.bulk(bulkRequest, RequestOptions.DEFAULT)
|
||||
|
||||
// 检查响应
|
||||
if (bulkResponse.hasFailures()) {
|
||||
println("批量更新中有失败的操作:")
|
||||
bulkResponse.buildFailureMessage().each { println(it) }
|
||||
} else {
|
||||
println("批量更新成功!")
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
// 关闭客户端
|
||||
try {
|
||||
client.close()
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// def queryPoiDatasByCity(){
|
||||
// def esUrl = "http://localhost:9200/your_index/_search"
|
||||
// def pageSize = 1000 // 每页返回的记录数
|
||||
// def scrollSize = '1m' // scroll上下文的生存时间
|
||||
//
|
||||
//// 初始化HttpClient和ObjectMapper
|
||||
// CloseableHttpClient httpClient = HttpClients.createDefault()
|
||||
// ObjectMapper objectMapper = new ObjectMapper()
|
||||
//
|
||||
//// 初始化scroll请求
|
||||
// def scrollRequest = """
|
||||
//{
|
||||
// "size": $pageSize,
|
||||
// "query": {
|
||||
// "match_all": {}
|
||||
// },
|
||||
// "scroll": "$scrollSize"
|
||||
//}
|
||||
//"""
|
||||
//
|
||||
//// 发送scroll初始化请求
|
||||
// HttpGet scrollInitRequest = new HttpGet(esUrl)
|
||||
// scrollInitRequest.setEntity(new org.apache.http.entity.StringEntity(scrollRequest, "UTF-8"))
|
||||
// scrollInitRequest.setHeader("Content-Type", "application/json")
|
||||
//
|
||||
// HttpResponse scrollInitResponse = httpClient.execute(scrollInitRequest)
|
||||
// HttpEntity entity = scrollInitResponse.getEntity()
|
||||
// String scrollId = objectMapper.readValue(EntityUtils.toString(entity), Map.class).get("_scroll_id").toString()
|
||||
//
|
||||
//// 存储所有结果
|
||||
// def allResults = []
|
||||
//
|
||||
//// 发送scroll请求获取数据,直到没有更多数据为止
|
||||
// while (true) {
|
||||
// def scrollRequestWithId = """
|
||||
// {
|
||||
// "scroll_id": "$scrollId",
|
||||
// "scroll": "$scrollSize"
|
||||
// }
|
||||
// """
|
||||
//
|
||||
// HttpGet scrollGetRequest = new HttpGet(esUrl + "?scroll")
|
||||
// scrollGetRequest.setEntity(new org.apache.http.entity.StringEntity(scrollRequestWithId, "UTF-8"))
|
||||
// scrollGetRequest.setHeader("Content-Type", "application/json")
|
||||
//
|
||||
// HttpResponse scrollGetResponse = httpClient.execute(scrollGetRequest)
|
||||
// entity = scrollGetResponse.getEntity()
|
||||
// def responseMap = objectMapper.readValue(EntityUtils.toString(entity), Map.class)
|
||||
//
|
||||
// def hits = responseMap.get("hits").get("hits")
|
||||
// if (hits.isEmpty()) {
|
||||
// break // 没有更多数据,跳出循环
|
||||
// }
|
||||
//
|
||||
// // 合并当前页的数据
|
||||
// allResults.addAll(hits.map({ it.get("_source") }))
|
||||
//
|
||||
// // 更新scrollId以进行下一次请求
|
||||
// scrollId = responseMap.get("_scroll_id").toString()
|
||||
// }
|
||||
//
|
||||
// // 关闭HttpClient
|
||||
// httpClient.close()
|
||||
//
|
||||
// // 输出或处理全量数据
|
||||
// println(allResults)
|
||||
//
|
||||
// // 注意:实际应用中,你可能需要在获取完数据后清除scroll上下文,以释放资源。
|
||||
// }
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user