查poi数据限制返回字段

This commit is contained in:
2024-11-04 16:45:39 +08:00
parent 482b620029
commit 10f0b79087
3 changed files with 48 additions and 77 deletions
@@ -29,8 +29,8 @@ class RegionStandardBatch extends BaseBatch{
@Override
void process(Object args) {
seaweedRegionStandard()
// poiRegionStandard()
// seaweedRegionStandard()
poiRegionStandard()
}
def seaweedRegionStandard(){
@@ -168,7 +168,8 @@ class RegionStandardBatch extends BaseBatch{
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("district", "蕉岭县"))
def rets = []
def datas = ElasticUtils89.executeSimpleQuery("192.168.0.89", "didata_map_poi", queryBuilder)
String[] includes = ["city", "district", "name", "township", "status", "id"]
def datas = ElasticUtils89.executeSimpleQuery("192.168.0.89", "didata_map_poi", queryBuilder,includes)
for(def d: datas){
def o = new BasicDBObject()
o.put("city", d.city)
@@ -1,5 +1,6 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.xiamen.ElasticUtils89
@@ -8,6 +9,8 @@ import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.update.UpdateRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.common.xcontent.XContentType
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
/**
* 区域标准化
@@ -20,6 +23,13 @@ class TownStandardBatch extends BaseBatch{
def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity()
def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity()
def poiEsUrl = "http://192.168.0.89:9200/didata_map_poi/_search"
def elasticsearchUrl = "http://192.168.0.89:9200/_search/scroll"
def initialSearchUrl = "http://192.168.0.89:9200/didata_map_poi/_search?scroll=1m"
def scrollTime = "1m" // Scroll context time-to-live
def pageSize = 1000 // Number of hits to return per scroll
static main(args) {
runBatch(new TownStandardBatch(), args)
}
@@ -28,11 +38,13 @@ class TownStandardBatch extends BaseBatch{
@Override
void process(Object args) {
TownStandard()
}
def TownStandard(){
townStandardByCat("office")
// townStandardByPoi()
// townStandardByCat("office")
townStandardByPoi()
}
def townStandardByCat(def cat){
//"三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德",
@@ -60,7 +72,7 @@ class TownStandardBatch extends BaseBatch{
}
def townStandardByPoi(){
def citys = ["重庆"]//CommonDatautils.findAllCity()
def citys = ["三明"]//CommonDatautils.findAllCity()
for(def city : citys){
def data = fullCityMap.get(city)
@@ -141,6 +153,9 @@ class TownStandardBatch extends BaseBatch{
}
}
def updateRegionByEs(rows){
def client = ElasticUtils89.getClient("192.168.0.89")
def bulkRequest = new BulkRequest()
@@ -177,75 +192,24 @@ class TownStandardBatch extends BaseBatch{
}
}
def findEsPoiByCity(city){
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
def rets = []
String[] includes = ["city", "district", "name", "township", "status", "id"]
def datas = ElasticUtils89.executeSimpleQuery("192.168.0.89", "didata_map_poi", queryBuilder, includes)
for(def d: datas){
def o = new BasicDBObject()
o.put("city", d.city)
o.put("district", d.district)
o.put("name", d.name)
o.put("township", d.township)
o.put("status", d.status)
o.put("id", d.id)
rets << o
}
return rets
}
// def queryPoiDatasByCity(){
// def esUrl = "http://localhost:9200/your_index/_search"
// def pageSize = 1000 // 每页返回的记录数
// def scrollSize = '1m' // scroll上下文的生存时间
//
//// 初始化HttpClient和ObjectMapper
// CloseableHttpClient httpClient = HttpClients.createDefault()
// ObjectMapper objectMapper = new ObjectMapper()
//
//// 初始化scroll请求
// def scrollRequest = """
//{
// "size": $pageSize,
// "query": {
// "match_all": {}
// },
// "scroll": "$scrollSize"
//}
//"""
//
//// 发送scroll初始化请求
// HttpGet scrollInitRequest = new HttpGet(esUrl)
// scrollInitRequest.setEntity(new org.apache.http.entity.StringEntity(scrollRequest, "UTF-8"))
// scrollInitRequest.setHeader("Content-Type", "application/json")
//
// HttpResponse scrollInitResponse = httpClient.execute(scrollInitRequest)
// HttpEntity entity = scrollInitResponse.getEntity()
// String scrollId = objectMapper.readValue(EntityUtils.toString(entity), Map.class).get("_scroll_id").toString()
//
//// 存储所有结果
// def allResults = []
//
//// 发送scroll请求获取数据,直到没有更多数据为止
// while (true) {
// def scrollRequestWithId = """
// {
// "scroll_id": "$scrollId",
// "scroll": "$scrollSize"
// }
// """
//
// HttpGet scrollGetRequest = new HttpGet(esUrl + "?scroll")
// scrollGetRequest.setEntity(new org.apache.http.entity.StringEntity(scrollRequestWithId, "UTF-8"))
// scrollGetRequest.setHeader("Content-Type", "application/json")
//
// HttpResponse scrollGetResponse = httpClient.execute(scrollGetRequest)
// entity = scrollGetResponse.getEntity()
// def responseMap = objectMapper.readValue(EntityUtils.toString(entity), Map.class)
//
// def hits = responseMap.get("hits").get("hits")
// if (hits.isEmpty()) {
// break // 没有更多数据,跳出循环
// }
//
// // 合并当前页的数据
// allResults.addAll(hits.map({ it.get("_source") }))
//
// // 更新scrollId以进行下一次请求
// scrollId = responseMap.get("_scroll_id").toString()
// }
//
// // 关闭HttpClient
// httpClient.close()
//
// // 输出或处理全量数据
// println(allResults)
//
// // 注意:实际应用中,你可能需要在获取完数据后清除scroll上下文,以释放资源。
// }
}
@@ -26,6 +26,7 @@ import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.search.Scroll
import org.elasticsearch.search.SearchHit
import org.elasticsearch.search.builder.SearchSourceBuilder
import org.elasticsearch.search.fetch.subphase.FetchSourceContext
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
@@ -35,7 +36,6 @@ import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
import java.util.function.BiConsumer
/**
* 功能说明:Elasticsearch工具类
*/
@@ -228,13 +228,19 @@ class ElasticUtils89 {
}
static def executeSimpleQuery(String host, String indexName, QueryBuilder qb){
static def executeSimpleQuery(String host, String indexName, QueryBuilder qb, includes){
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L))
RestHighLevelClient client = getClient(host)
SearchRequest searchRequest = new SearchRequest(indexName);
searchRequest.scroll(scroll);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
if(includes){
String[] excludes = [] // 没有要排除的字段
FetchSourceContext fetchSourceContext = new FetchSourceContext(true, includes, excludes)
searchSourceBuilder.fetchSource(fetchSourceContext)
}
searchSourceBuilder.query(qb);
// searchSourceBuilder.size(1000)
searchRequest.source(searchSourceBuilder);