From b3a1114cc4e3cd11dd102597938a329530d31c11 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 1 Nov 2024 11:01:55 +0800 Subject: [PATCH] =?UTF-8?q?=E4=B9=A1=E9=95=87=E8=A1=97=E9=81=93=E6=A0=87?= =?UTF-8?q?=E5=87=86=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gj/data_handle/TownStandardBatch.groovy | 251 ++++++++++++++++++ 1 file changed, 251 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy new file mode 100644 index 000000000..afe90bbb2 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy @@ -0,0 +1,251 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.BaseBatch +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.xiamen.ElasticUtils89 +import com.mongodb.BasicDBObject +import org.elasticsearch.action.bulk.BulkRequest +import org.elasticsearch.action.update.UpdateRequest +import org.elasticsearch.client.RequestOptions +import org.elasticsearch.common.xcontent.XContentType + +/** + * 区域标准化 + * 1.小区库乡镇街道标准化,包含小区,商铺,写字楼 + * 2.poi数据库中乡镇街道标准化 + */ +class TownStandardBatch extends BaseBatch{ + + def fullCityMap = CommonDatautils.queryFullNameCity() + def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity() + def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity() + + static main(args) { + runBatch(new TownStandardBatch(), args) + } + + + @Override + void process(Object args) { + TownStandard() + } + + def TownStandard(){ + townStandardByCat("office") +// townStandardByPoi() + } + def townStandardByCat(def cat){ + //"三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德", + def citys = CommonDatautils.findAllCity() + for(def city : citys){ + + def data = fullCityMap.get(city) + def fullCityName = null + if(data){ + fullCityName = data.city + } + + def mzbRegionMap = CommonDatautils.queryMzbRegion(fullCityName) + def shortRegionNameMap = CommonDatautils.getRegionShortName(fullCityName) + def townMap = CommonDatautils.queryTownByCity(fullCityName, ["city","district", "street", "alias"], mzbRegionMap) + if(townMap){ + def regionSet = townMap.keySet() + if(regionSet.size()>1){ + haveRegionProcess(city, fullCityName, townMap, cat, shortRegionNameMap) + }else{ + println city +" 无区域" + } + } + } + } + + def townStandardByPoi(){ + def citys = ["重庆"]//CommonDatautils.findAllCity() + for(def city : citys){ + + def data = fullCityMap.get(city) + def fullCityName = null + if(data){ + fullCityName = data.city + } + if(fullCityName){ + + def rMap = regionFullNameMap.get(fullCityName) + if(rMap){ + def set = rMap.keySet() + def rets = findEsPoiByCity(fullCityName) + println city +" "+rets.size() + if(set.size()>1){ + poiHaveRegionProcess(city, set, rMap, rets) + }else{ +// haveStreetProcess(city, set, cat) + } + } + }else{ + println "没找到相应的全称:"+city + } + } + } + def haveRegionProcess(city, fullCityName, townMap, cat, shortRegionNameMap){ + def results = [] + if(fullCityName){ + def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name", "township"]) + for(def d : districts){ + if(d.township){ + def towns = null + def updateTown = null + for(def m : townMap){ + def fullRegionName = shortRegionNameMap.get(d.region) + if(m.key == fullRegionName){ + towns = m.value + break + } + } + + def opt = true + if(towns){ + for(def town : towns){ + def townAlias = [] + if(town.alias){ + townAlias = town.alias + } + townAlias << town.street + if(townAlias.contains(d.township)){ + if(d.township!=town.street){ + updateTown = town.street + }else{ + opt = false + } + break + } + } + } + if(opt){ + if(updateTown){ + def udata = new BasicDBObject() + udata.put("township", updateTown) + udata.put("m_date", new Date()) + println d.city +" "+d.region+" "+d.name+" "+d.township +" 乡镇修改为:"+updateTown +// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata)) + }else{ + def udata = new BasicDBObject() + udata.put("township", "") + udata.put("m_date", new Date()) + println d.city +" "+d.region+" "+d.name+" "+d.township +" 乡镇修改为:空值" +// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata)) + } + } + + } + } + } + } + + def updateRegionByEs(rows){ + def client = ElasticUtils89.getClient("192.168.0.89") + def bulkRequest = new BulkRequest() + + rows.each { row -> + def id = row.keySet().iterator().next() + def doc = row.get(id) + + def updateRequest = new UpdateRequest("didata_map_poi", id) + updateRequest.doc(doc, XContentType.JSON) + bulkRequest.add(updateRequest) + } + + try { + // 执行批量更新请求 + def bulkResponse = client.bulk(bulkRequest, RequestOptions.DEFAULT) + + // 检查响应 + if (bulkResponse.hasFailures()) { + println("批量更新中有失败的操作:") + bulkResponse.buildFailureMessage().each { println(it) } + } else { + println("批量更新成功!") + } + } catch (Exception e) { + e.printStackTrace() + } finally { + // 关闭客户端 + try { + client.close() + } catch (Exception e) { + e.printStackTrace() + } + } + } + + + // def queryPoiDatasByCity(){ +// def esUrl = "http://localhost:9200/your_index/_search" +// def pageSize = 1000 // 每页返回的记录数 +// def scrollSize = '1m' // scroll上下文的生存时间 +// +//// 初始化HttpClient和ObjectMapper +// CloseableHttpClient httpClient = HttpClients.createDefault() +// ObjectMapper objectMapper = new ObjectMapper() +// +//// 初始化scroll请求 +// def scrollRequest = """ +//{ +// "size": $pageSize, +// "query": { +// "match_all": {} +// }, +// "scroll": "$scrollSize" +//} +//""" +// +//// 发送scroll初始化请求 +// HttpGet scrollInitRequest = new HttpGet(esUrl) +// scrollInitRequest.setEntity(new org.apache.http.entity.StringEntity(scrollRequest, "UTF-8")) +// scrollInitRequest.setHeader("Content-Type", "application/json") +// +// HttpResponse scrollInitResponse = httpClient.execute(scrollInitRequest) +// HttpEntity entity = scrollInitResponse.getEntity() +// String scrollId = objectMapper.readValue(EntityUtils.toString(entity), Map.class).get("_scroll_id").toString() +// +//// 存储所有结果 +// def allResults = [] +// +//// 发送scroll请求获取数据,直到没有更多数据为止 +// while (true) { +// def scrollRequestWithId = """ +// { +// "scroll_id": "$scrollId", +// "scroll": "$scrollSize" +// } +// """ +// +// HttpGet scrollGetRequest = new HttpGet(esUrl + "?scroll") +// scrollGetRequest.setEntity(new org.apache.http.entity.StringEntity(scrollRequestWithId, "UTF-8")) +// scrollGetRequest.setHeader("Content-Type", "application/json") +// +// HttpResponse scrollGetResponse = httpClient.execute(scrollGetRequest) +// entity = scrollGetResponse.getEntity() +// def responseMap = objectMapper.readValue(EntityUtils.toString(entity), Map.class) +// +// def hits = responseMap.get("hits").get("hits") +// if (hits.isEmpty()) { +// break // 没有更多数据,跳出循环 +// } +// +// // 合并当前页的数据 +// allResults.addAll(hits.map({ it.get("_source") })) +// +// // 更新scrollId以进行下一次请求 +// scrollId = responseMap.get("_scroll_id").toString() +// } +// +// // 关闭HttpClient +// httpClient.close() +// +// // 输出或处理全量数据 +// println(allResults) +// +// // 注意:实际应用中,你可能需要在获取完数据后清除scroll上下文,以释放资源。 +// } + +}