From 8deebf69c5d938b7a000478ea2c593d943c2417a Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sun, 10 Nov 2024 10:32:24 +0800 Subject: [PATCH] updadte --- .../gj/data_handle/RegionStandardBatch.groovy | 76 ++++++++++++++++++- 1 file changed, 72 insertions(+), 4 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy index aeac02b63..fc55cbd77 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy @@ -12,7 +12,6 @@ import org.elasticsearch.client.RequestOptions import org.elasticsearch.common.xcontent.XContentType import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilders - /** * 区域标准化 * 1.小区库区域标准化,包含小区,商铺,写字楼 @@ -163,7 +162,7 @@ class RegionStandardBatch extends BaseBatch{ } def findEsPoiByCity(city){ - QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)) + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("district", "蕉岭县")) def rets = [] def datas = ElasticUtils89.executeSimpleQuery("192.168.0.89", "didata_map_poi", queryBuilder) @@ -200,7 +199,7 @@ class RegionStandardBatch extends BaseBatch{ def m = [:] m.put(d.id, ["district":updateRegion]) results << m - if(results.size()%1==0){ + if(results.size()%100==0){ updateRegionByEs(results) results.clear() } @@ -220,7 +219,7 @@ class RegionStandardBatch extends BaseBatch{ } } - if(results.size()%1>0){ + if(results.size()>0){ updateRegionByEs(results) results.clear() } @@ -265,6 +264,75 @@ class RegionStandardBatch extends BaseBatch{ } +// def queryPoiDatasByCity(){ +// def esUrl = "http://localhost:9200/your_index/_search" +// def pageSize = 1000 // 每页返回的记录数 +// def scrollSize = '1m' // scroll上下文的生存时间 +// +//// 初始化HttpClient和ObjectMapper +// CloseableHttpClient httpClient = HttpClients.createDefault() +// ObjectMapper objectMapper = new ObjectMapper() +// +//// 初始化scroll请求 +// def scrollRequest = """ +//{ +// "size": $pageSize, +// "query": { +// "match_all": {} +// }, +// "scroll": "$scrollSize" +//} +//""" +// +//// 发送scroll初始化请求 +// HttpGet scrollInitRequest = new HttpGet(esUrl) +// scrollInitRequest.setEntity(new org.apache.http.entity.StringEntity(scrollRequest, "UTF-8")) +// scrollInitRequest.setHeader("Content-Type", "application/json") +// +// HttpResponse scrollInitResponse = httpClient.execute(scrollInitRequest) +// HttpEntity entity = scrollInitResponse.getEntity() +// String scrollId = objectMapper.readValue(EntityUtils.toString(entity), Map.class).get("_scroll_id").toString() +// +//// 存储所有结果 +// def allResults = [] +// +//// 发送scroll请求获取数据,直到没有更多数据为止 +// while (true) { +// def scrollRequestWithId = """ +// { +// "scroll_id": "$scrollId", +// "scroll": "$scrollSize" +// } +// """ +// +// HttpGet scrollGetRequest = new HttpGet(esUrl + "?scroll") +// scrollGetRequest.setEntity(new org.apache.http.entity.StringEntity(scrollRequestWithId, "UTF-8")) +// scrollGetRequest.setHeader("Content-Type", "application/json") +// +// HttpResponse scrollGetResponse = httpClient.execute(scrollGetRequest) +// entity = scrollGetResponse.getEntity() +// def responseMap = objectMapper.readValue(EntityUtils.toString(entity), Map.class) +// +// def hits = responseMap.get("hits").get("hits") +// if (hits.isEmpty()) { +// break // 没有更多数据,跳出循环 +// } +// +// // 合并当前页的数据 +// allResults.addAll(hits.map({ it.get("_source") })) +// +// // 更新scrollId以进行下一次请求 +// scrollId = responseMap.get("_scroll_id").toString() +// } +// +// // 关闭HttpClient +// httpClient.close() +// +// // 输出或处理全量数据 +// println(allResults) +// +// // 注意:实际应用中,你可能需要在获取完数据后清除scroll上下文,以释放资源。 +// } }