This commit is contained in:
2024-11-10 10:32:24 +08:00
parent 2be600c140
commit 8deebf69c5
@@ -12,7 +12,6 @@ import org.elasticsearch.client.RequestOptions
import org.elasticsearch.common.xcontent.XContentType
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
/**
* 区域标准化
* 1.小区库区域标准化,包含小区,商铺,写字楼
@@ -163,7 +162,7 @@ class RegionStandardBatch extends BaseBatch{
}
def findEsPoiByCity(city){
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("district", "蕉岭县"))
def rets = []
def datas = ElasticUtils89.executeSimpleQuery("192.168.0.89", "didata_map_poi", queryBuilder)
@@ -200,7 +199,7 @@ class RegionStandardBatch extends BaseBatch{
def m = [:]
m.put(d.id, ["district":updateRegion])
results << m
if(results.size()%1==0){
if(results.size()%100==0){
updateRegionByEs(results)
results.clear()
}
@@ -220,7 +219,7 @@ class RegionStandardBatch extends BaseBatch{
}
}
if(results.size()%1>0){
if(results.size()>0){
updateRegionByEs(results)
results.clear()
}
@@ -265,6 +264,75 @@ class RegionStandardBatch extends BaseBatch{
}
// def queryPoiDatasByCity(){
// def esUrl = "http://localhost:9200/your_index/_search"
// def pageSize = 1000 // 每页返回的记录数
// def scrollSize = '1m' // scroll上下文的生存时间
//
//// 初始化HttpClient和ObjectMapper
// CloseableHttpClient httpClient = HttpClients.createDefault()
// ObjectMapper objectMapper = new ObjectMapper()
//
//// 初始化scroll请求
// def scrollRequest = """
//{
// "size": $pageSize,
// "query": {
// "match_all": {}
// },
// "scroll": "$scrollSize"
//}
//"""
//
//// 发送scroll初始化请求
// HttpGet scrollInitRequest = new HttpGet(esUrl)
// scrollInitRequest.setEntity(new org.apache.http.entity.StringEntity(scrollRequest, "UTF-8"))
// scrollInitRequest.setHeader("Content-Type", "application/json")
//
// HttpResponse scrollInitResponse = httpClient.execute(scrollInitRequest)
// HttpEntity entity = scrollInitResponse.getEntity()
// String scrollId = objectMapper.readValue(EntityUtils.toString(entity), Map.class).get("_scroll_id").toString()
//
//// 存储所有结果
// def allResults = []
//
//// 发送scroll请求获取数据,直到没有更多数据为止
// while (true) {
// def scrollRequestWithId = """
// {
// "scroll_id": "$scrollId",
// "scroll": "$scrollSize"
// }
// """
//
// HttpGet scrollGetRequest = new HttpGet(esUrl + "?scroll")
// scrollGetRequest.setEntity(new org.apache.http.entity.StringEntity(scrollRequestWithId, "UTF-8"))
// scrollGetRequest.setHeader("Content-Type", "application/json")
//
// HttpResponse scrollGetResponse = httpClient.execute(scrollGetRequest)
// entity = scrollGetResponse.getEntity()
// def responseMap = objectMapper.readValue(EntityUtils.toString(entity), Map.class)
//
// def hits = responseMap.get("hits").get("hits")
// if (hits.isEmpty()) {
// break // 没有更多数据,跳出循环
// }
//
// // 合并当前页的数据
// allResults.addAll(hits.map({ it.get("_source") }))
//
// // 更新scrollId以进行下一次请求
// scrollId = responseMap.get("_scroll_id").toString()
// }
//
// // 关闭HttpClient
// httpClient.close()
//
// // 输出或处理全量数据
// println(allResults)
//
// // 注意:实际应用中,你可能需要在获取完数据后清除scroll上下文,以释放资源。
// }
}