diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy index 12213a4b9..9e30b064c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy @@ -29,8 +29,8 @@ class RegionStandardBatch extends BaseBatch{ @Override void process(Object args) { - seaweedRegionStandard() -// poiRegionStandard() +// seaweedRegionStandard() + poiRegionStandard() } def seaweedRegionStandard(){ @@ -168,7 +168,8 @@ class RegionStandardBatch extends BaseBatch{ QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("district", "蕉岭县")) def rets = [] - def datas = ElasticUtils89.executeSimpleQuery("192.168.0.89", "didata_map_poi", queryBuilder) + String[] includes = ["city", "district", "name", "township", "status", "id"] + def datas = ElasticUtils89.executeSimpleQuery("192.168.0.89", "didata_map_poi", queryBuilder,includes) for(def d: datas){ def o = new BasicDBObject() o.put("city", d.city) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy index afe90bbb2..d2c6381ae 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.gj.data_handle + import com.fangjia.script.BaseBatch import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.xiamen.ElasticUtils89 @@ -8,6 +9,8 @@ import org.elasticsearch.action.bulk.BulkRequest import org.elasticsearch.action.update.UpdateRequest import org.elasticsearch.client.RequestOptions import org.elasticsearch.common.xcontent.XContentType +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders /** * 区域标准化 @@ -20,6 +23,13 @@ class TownStandardBatch extends BaseBatch{ def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity() def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity() + def poiEsUrl = "http://192.168.0.89:9200/didata_map_poi/_search" + + def elasticsearchUrl = "http://192.168.0.89:9200/_search/scroll" + def initialSearchUrl = "http://192.168.0.89:9200/didata_map_poi/_search?scroll=1m" + def scrollTime = "1m" // Scroll context time-to-live + def pageSize = 1000 // Number of hits to return per scroll + static main(args) { runBatch(new TownStandardBatch(), args) } @@ -28,11 +38,13 @@ class TownStandardBatch extends BaseBatch{ @Override void process(Object args) { TownStandard() + + } def TownStandard(){ - townStandardByCat("office") -// townStandardByPoi() +// townStandardByCat("office") + townStandardByPoi() } def townStandardByCat(def cat){ //"三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德", @@ -60,7 +72,7 @@ class TownStandardBatch extends BaseBatch{ } def townStandardByPoi(){ - def citys = ["重庆"]//CommonDatautils.findAllCity() + def citys = ["三明"]//CommonDatautils.findAllCity() for(def city : citys){ def data = fullCityMap.get(city) @@ -141,6 +153,9 @@ class TownStandardBatch extends BaseBatch{ } } + + + def updateRegionByEs(rows){ def client = ElasticUtils89.getClient("192.168.0.89") def bulkRequest = new BulkRequest() @@ -177,75 +192,24 @@ class TownStandardBatch extends BaseBatch{ } } + def findEsPoiByCity(city){ + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)) + def rets = [] + + String[] includes = ["city", "district", "name", "township", "status", "id"] + def datas = ElasticUtils89.executeSimpleQuery("192.168.0.89", "didata_map_poi", queryBuilder, includes) + for(def d: datas){ + def o = new BasicDBObject() + o.put("city", d.city) + o.put("district", d.district) + o.put("name", d.name) + o.put("township", d.township) + o.put("status", d.status) + o.put("id", d.id) + rets << o + } + return rets + } - // def queryPoiDatasByCity(){ -// def esUrl = "http://localhost:9200/your_index/_search" -// def pageSize = 1000 // 每页返回的记录数 -// def scrollSize = '1m' // scroll上下文的生存时间 -// -//// 初始化HttpClient和ObjectMapper -// CloseableHttpClient httpClient = HttpClients.createDefault() -// ObjectMapper objectMapper = new ObjectMapper() -// -//// 初始化scroll请求 -// def scrollRequest = """ -//{ -// "size": $pageSize, -// "query": { -// "match_all": {} -// }, -// "scroll": "$scrollSize" -//} -//""" -// -//// 发送scroll初始化请求 -// HttpGet scrollInitRequest = new HttpGet(esUrl) -// scrollInitRequest.setEntity(new org.apache.http.entity.StringEntity(scrollRequest, "UTF-8")) -// scrollInitRequest.setHeader("Content-Type", "application/json") -// -// HttpResponse scrollInitResponse = httpClient.execute(scrollInitRequest) -// HttpEntity entity = scrollInitResponse.getEntity() -// String scrollId = objectMapper.readValue(EntityUtils.toString(entity), Map.class).get("_scroll_id").toString() -// -//// 存储所有结果 -// def allResults = [] -// -//// 发送scroll请求获取数据,直到没有更多数据为止 -// while (true) { -// def scrollRequestWithId = """ -// { -// "scroll_id": "$scrollId", -// "scroll": "$scrollSize" -// } -// """ -// -// HttpGet scrollGetRequest = new HttpGet(esUrl + "?scroll") -// scrollGetRequest.setEntity(new org.apache.http.entity.StringEntity(scrollRequestWithId, "UTF-8")) -// scrollGetRequest.setHeader("Content-Type", "application/json") -// -// HttpResponse scrollGetResponse = httpClient.execute(scrollGetRequest) -// entity = scrollGetResponse.getEntity() -// def responseMap = objectMapper.readValue(EntityUtils.toString(entity), Map.class) -// -// def hits = responseMap.get("hits").get("hits") -// if (hits.isEmpty()) { -// break // 没有更多数据,跳出循环 -// } -// -// // 合并当前页的数据 -// allResults.addAll(hits.map({ it.get("_source") })) -// -// // 更新scrollId以进行下一次请求 -// scrollId = responseMap.get("_scroll_id").toString() -// } -// -// // 关闭HttpClient -// httpClient.close() -// -// // 输出或处理全量数据 -// println(allResults) -// -// // 注意:实际应用中,你可能需要在获取完数据后清除scroll上下文,以释放资源。 -// } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy index e31ded3ec..a401bf480 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy @@ -26,6 +26,7 @@ import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.search.Scroll import org.elasticsearch.search.SearchHit import org.elasticsearch.search.builder.SearchSourceBuilder +import org.elasticsearch.search.fetch.subphase.FetchSourceContext import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.sql.Connection @@ -35,7 +36,6 @@ import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.TimeUnit import java.util.function.BiConsumer - /** * 功能说明:Elasticsearch工具类 */ @@ -228,13 +228,19 @@ class ElasticUtils89 { } - static def executeSimpleQuery(String host, String indexName, QueryBuilder qb){ + static def executeSimpleQuery(String host, String indexName, QueryBuilder qb, includes){ final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L)) RestHighLevelClient client = getClient(host) SearchRequest searchRequest = new SearchRequest(indexName); searchRequest.scroll(scroll); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + if(includes){ + String[] excludes = [] // 没有要排除的字段 + FetchSourceContext fetchSourceContext = new FetchSourceContext(true, includes, excludes) + searchSourceBuilder.fetchSource(fetchSourceContext) + } + searchSourceBuilder.query(qb); // searchSourceBuilder.size(1000) searchRequest.source(searchSourceBuilder);