Merge branch 'script_gj' into 'master'
查poi数据限制返回字段 See merge request !994
This commit is contained in:
+4
-3
@@ -29,8 +29,8 @@ class RegionStandardBatch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
seaweedRegionStandard()
|
||||
// poiRegionStandard()
|
||||
// seaweedRegionStandard()
|
||||
poiRegionStandard()
|
||||
}
|
||||
|
||||
def seaweedRegionStandard(){
|
||||
@@ -168,7 +168,8 @@ class RegionStandardBatch extends BaseBatch{
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("district", "蕉岭县"))
|
||||
def rets = []
|
||||
|
||||
def datas = ElasticUtils89.executeSimpleQuery("192.168.0.89", "didata_map_poi", queryBuilder)
|
||||
String[] includes = ["city", "district", "name", "township", "status", "id"]
|
||||
def datas = ElasticUtils89.executeSimpleQuery("192.168.0.89", "didata_map_poi", queryBuilder,includes)
|
||||
for(def d: datas){
|
||||
def o = new BasicDBObject()
|
||||
o.put("city", d.city)
|
||||
|
||||
+36
-72
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
||||
@@ -8,6 +9,8 @@ import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.update.UpdateRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.common.xcontent.XContentType
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
/**
|
||||
* 区域标准化
|
||||
@@ -20,6 +23,13 @@ class TownStandardBatch extends BaseBatch{
|
||||
def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity()
|
||||
def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity()
|
||||
|
||||
def poiEsUrl = "http://192.168.0.89:9200/didata_map_poi/_search"
|
||||
|
||||
def elasticsearchUrl = "http://192.168.0.89:9200/_search/scroll"
|
||||
def initialSearchUrl = "http://192.168.0.89:9200/didata_map_poi/_search?scroll=1m"
|
||||
def scrollTime = "1m" // Scroll context time-to-live
|
||||
def pageSize = 1000 // Number of hits to return per scroll
|
||||
|
||||
static main(args) {
|
||||
runBatch(new TownStandardBatch(), args)
|
||||
}
|
||||
@@ -28,11 +38,13 @@ class TownStandardBatch extends BaseBatch{
|
||||
@Override
|
||||
void process(Object args) {
|
||||
TownStandard()
|
||||
|
||||
|
||||
}
|
||||
|
||||
def TownStandard(){
|
||||
townStandardByCat("office")
|
||||
// townStandardByPoi()
|
||||
// townStandardByCat("office")
|
||||
townStandardByPoi()
|
||||
}
|
||||
def townStandardByCat(def cat){
|
||||
//"三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德",
|
||||
@@ -60,7 +72,7 @@ class TownStandardBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
def townStandardByPoi(){
|
||||
def citys = ["重庆"]//CommonDatautils.findAllCity()
|
||||
def citys = ["三明"]//CommonDatautils.findAllCity()
|
||||
for(def city : citys){
|
||||
|
||||
def data = fullCityMap.get(city)
|
||||
@@ -141,6 +153,9 @@ class TownStandardBatch extends BaseBatch{
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
def updateRegionByEs(rows){
|
||||
def client = ElasticUtils89.getClient("192.168.0.89")
|
||||
def bulkRequest = new BulkRequest()
|
||||
@@ -177,75 +192,24 @@ class TownStandardBatch extends BaseBatch{
|
||||
}
|
||||
}
|
||||
|
||||
def findEsPoiByCity(city){
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
|
||||
def rets = []
|
||||
|
||||
String[] includes = ["city", "district", "name", "township", "status", "id"]
|
||||
def datas = ElasticUtils89.executeSimpleQuery("192.168.0.89", "didata_map_poi", queryBuilder, includes)
|
||||
for(def d: datas){
|
||||
def o = new BasicDBObject()
|
||||
o.put("city", d.city)
|
||||
o.put("district", d.district)
|
||||
o.put("name", d.name)
|
||||
o.put("township", d.township)
|
||||
o.put("status", d.status)
|
||||
o.put("id", d.id)
|
||||
rets << o
|
||||
}
|
||||
return rets
|
||||
}
|
||||
|
||||
// def queryPoiDatasByCity(){
|
||||
// def esUrl = "http://localhost:9200/your_index/_search"
|
||||
// def pageSize = 1000 // 每页返回的记录数
|
||||
// def scrollSize = '1m' // scroll上下文的生存时间
|
||||
//
|
||||
//// 初始化HttpClient和ObjectMapper
|
||||
// CloseableHttpClient httpClient = HttpClients.createDefault()
|
||||
// ObjectMapper objectMapper = new ObjectMapper()
|
||||
//
|
||||
//// 初始化scroll请求
|
||||
// def scrollRequest = """
|
||||
//{
|
||||
// "size": $pageSize,
|
||||
// "query": {
|
||||
// "match_all": {}
|
||||
// },
|
||||
// "scroll": "$scrollSize"
|
||||
//}
|
||||
//"""
|
||||
//
|
||||
//// 发送scroll初始化请求
|
||||
// HttpGet scrollInitRequest = new HttpGet(esUrl)
|
||||
// scrollInitRequest.setEntity(new org.apache.http.entity.StringEntity(scrollRequest, "UTF-8"))
|
||||
// scrollInitRequest.setHeader("Content-Type", "application/json")
|
||||
//
|
||||
// HttpResponse scrollInitResponse = httpClient.execute(scrollInitRequest)
|
||||
// HttpEntity entity = scrollInitResponse.getEntity()
|
||||
// String scrollId = objectMapper.readValue(EntityUtils.toString(entity), Map.class).get("_scroll_id").toString()
|
||||
//
|
||||
//// 存储所有结果
|
||||
// def allResults = []
|
||||
//
|
||||
//// 发送scroll请求获取数据,直到没有更多数据为止
|
||||
// while (true) {
|
||||
// def scrollRequestWithId = """
|
||||
// {
|
||||
// "scroll_id": "$scrollId",
|
||||
// "scroll": "$scrollSize"
|
||||
// }
|
||||
// """
|
||||
//
|
||||
// HttpGet scrollGetRequest = new HttpGet(esUrl + "?scroll")
|
||||
// scrollGetRequest.setEntity(new org.apache.http.entity.StringEntity(scrollRequestWithId, "UTF-8"))
|
||||
// scrollGetRequest.setHeader("Content-Type", "application/json")
|
||||
//
|
||||
// HttpResponse scrollGetResponse = httpClient.execute(scrollGetRequest)
|
||||
// entity = scrollGetResponse.getEntity()
|
||||
// def responseMap = objectMapper.readValue(EntityUtils.toString(entity), Map.class)
|
||||
//
|
||||
// def hits = responseMap.get("hits").get("hits")
|
||||
// if (hits.isEmpty()) {
|
||||
// break // 没有更多数据,跳出循环
|
||||
// }
|
||||
//
|
||||
// // 合并当前页的数据
|
||||
// allResults.addAll(hits.map({ it.get("_source") }))
|
||||
//
|
||||
// // 更新scrollId以进行下一次请求
|
||||
// scrollId = responseMap.get("_scroll_id").toString()
|
||||
// }
|
||||
//
|
||||
// // 关闭HttpClient
|
||||
// httpClient.close()
|
||||
//
|
||||
// // 输出或处理全量数据
|
||||
// println(allResults)
|
||||
//
|
||||
// // 注意:实际应用中,你可能需要在获取完数据后清除scroll上下文,以释放资源。
|
||||
// }
|
||||
|
||||
}
|
||||
|
||||
+8
-2
@@ -26,6 +26,7 @@ import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.search.Scroll
|
||||
import org.elasticsearch.search.SearchHit
|
||||
import org.elasticsearch.search.builder.SearchSourceBuilder
|
||||
import org.elasticsearch.search.fetch.subphase.FetchSourceContext
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
@@ -35,7 +36,6 @@ import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.TimeUnit
|
||||
import java.util.function.BiConsumer
|
||||
|
||||
/**
|
||||
* 功能说明:Elasticsearch工具类
|
||||
*/
|
||||
@@ -228,13 +228,19 @@ class ElasticUtils89 {
|
||||
}
|
||||
|
||||
|
||||
static def executeSimpleQuery(String host, String indexName, QueryBuilder qb){
|
||||
static def executeSimpleQuery(String host, String indexName, QueryBuilder qb, includes){
|
||||
|
||||
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L))
|
||||
RestHighLevelClient client = getClient(host)
|
||||
SearchRequest searchRequest = new SearchRequest(indexName);
|
||||
searchRequest.scroll(scroll);
|
||||
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
|
||||
if(includes){
|
||||
String[] excludes = [] // 没有要排除的字段
|
||||
FetchSourceContext fetchSourceContext = new FetchSourceContext(true, includes, excludes)
|
||||
searchSourceBuilder.fetchSource(fetchSourceContext)
|
||||
}
|
||||
|
||||
searchSourceBuilder.query(qb);
|
||||
// searchSourceBuilder.size(1000)
|
||||
searchRequest.source(searchSourceBuilder);
|
||||
|
||||
Reference in New Issue
Block a user