updadte
This commit is contained in:
+72
-4
@@ -12,7 +12,6 @@ import org.elasticsearch.client.RequestOptions
|
|||||||
import org.elasticsearch.common.xcontent.XContentType
|
import org.elasticsearch.common.xcontent.XContentType
|
||||||
import org.elasticsearch.index.query.QueryBuilder
|
import org.elasticsearch.index.query.QueryBuilder
|
||||||
import org.elasticsearch.index.query.QueryBuilders
|
import org.elasticsearch.index.query.QueryBuilders
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 区域标准化
|
* 区域标准化
|
||||||
* 1.小区库区域标准化,包含小区,商铺,写字楼
|
* 1.小区库区域标准化,包含小区,商铺,写字楼
|
||||||
@@ -163,7 +162,7 @@ class RegionStandardBatch extends BaseBatch{
|
|||||||
}
|
}
|
||||||
|
|
||||||
def findEsPoiByCity(city){
|
def findEsPoiByCity(city){
|
||||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
|
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("district", "蕉岭县"))
|
||||||
def rets = []
|
def rets = []
|
||||||
|
|
||||||
def datas = ElasticUtils89.executeSimpleQuery("192.168.0.89", "didata_map_poi", queryBuilder)
|
def datas = ElasticUtils89.executeSimpleQuery("192.168.0.89", "didata_map_poi", queryBuilder)
|
||||||
@@ -200,7 +199,7 @@ class RegionStandardBatch extends BaseBatch{
|
|||||||
def m = [:]
|
def m = [:]
|
||||||
m.put(d.id, ["district":updateRegion])
|
m.put(d.id, ["district":updateRegion])
|
||||||
results << m
|
results << m
|
||||||
if(results.size()%1==0){
|
if(results.size()%100==0){
|
||||||
updateRegionByEs(results)
|
updateRegionByEs(results)
|
||||||
results.clear()
|
results.clear()
|
||||||
}
|
}
|
||||||
@@ -220,7 +219,7 @@ class RegionStandardBatch extends BaseBatch{
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if(results.size()%1>0){
|
if(results.size()>0){
|
||||||
updateRegionByEs(results)
|
updateRegionByEs(results)
|
||||||
results.clear()
|
results.clear()
|
||||||
}
|
}
|
||||||
@@ -265,6 +264,75 @@ class RegionStandardBatch extends BaseBatch{
|
|||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// def queryPoiDatasByCity(){
|
||||||
|
// def esUrl = "http://localhost:9200/your_index/_search"
|
||||||
|
// def pageSize = 1000 // 每页返回的记录数
|
||||||
|
// def scrollSize = '1m' // scroll上下文的生存时间
|
||||||
|
//
|
||||||
|
//// 初始化HttpClient和ObjectMapper
|
||||||
|
// CloseableHttpClient httpClient = HttpClients.createDefault()
|
||||||
|
// ObjectMapper objectMapper = new ObjectMapper()
|
||||||
|
//
|
||||||
|
//// 初始化scroll请求
|
||||||
|
// def scrollRequest = """
|
||||||
|
//{
|
||||||
|
// "size": $pageSize,
|
||||||
|
// "query": {
|
||||||
|
// "match_all": {}
|
||||||
|
// },
|
||||||
|
// "scroll": "$scrollSize"
|
||||||
|
//}
|
||||||
|
//"""
|
||||||
|
//
|
||||||
|
//// 发送scroll初始化请求
|
||||||
|
// HttpGet scrollInitRequest = new HttpGet(esUrl)
|
||||||
|
// scrollInitRequest.setEntity(new org.apache.http.entity.StringEntity(scrollRequest, "UTF-8"))
|
||||||
|
// scrollInitRequest.setHeader("Content-Type", "application/json")
|
||||||
|
//
|
||||||
|
// HttpResponse scrollInitResponse = httpClient.execute(scrollInitRequest)
|
||||||
|
// HttpEntity entity = scrollInitResponse.getEntity()
|
||||||
|
// String scrollId = objectMapper.readValue(EntityUtils.toString(entity), Map.class).get("_scroll_id").toString()
|
||||||
|
//
|
||||||
|
//// 存储所有结果
|
||||||
|
// def allResults = []
|
||||||
|
//
|
||||||
|
//// 发送scroll请求获取数据,直到没有更多数据为止
|
||||||
|
// while (true) {
|
||||||
|
// def scrollRequestWithId = """
|
||||||
|
// {
|
||||||
|
// "scroll_id": "$scrollId",
|
||||||
|
// "scroll": "$scrollSize"
|
||||||
|
// }
|
||||||
|
// """
|
||||||
|
//
|
||||||
|
// HttpGet scrollGetRequest = new HttpGet(esUrl + "?scroll")
|
||||||
|
// scrollGetRequest.setEntity(new org.apache.http.entity.StringEntity(scrollRequestWithId, "UTF-8"))
|
||||||
|
// scrollGetRequest.setHeader("Content-Type", "application/json")
|
||||||
|
//
|
||||||
|
// HttpResponse scrollGetResponse = httpClient.execute(scrollGetRequest)
|
||||||
|
// entity = scrollGetResponse.getEntity()
|
||||||
|
// def responseMap = objectMapper.readValue(EntityUtils.toString(entity), Map.class)
|
||||||
|
//
|
||||||
|
// def hits = responseMap.get("hits").get("hits")
|
||||||
|
// if (hits.isEmpty()) {
|
||||||
|
// break // 没有更多数据,跳出循环
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
// // 合并当前页的数据
|
||||||
|
// allResults.addAll(hits.map({ it.get("_source") }))
|
||||||
|
//
|
||||||
|
// // 更新scrollId以进行下一次请求
|
||||||
|
// scrollId = responseMap.get("_scroll_id").toString()
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
// // 关闭HttpClient
|
||||||
|
// httpClient.close()
|
||||||
|
//
|
||||||
|
// // 输出或处理全量数据
|
||||||
|
// println(allResults)
|
||||||
|
//
|
||||||
|
// // 注意:实际应用中,你可能需要在获取完数据后清除scroll上下文,以释放资源。
|
||||||
|
// }
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user