From 1626ab944d0483d65eb7af1759bff96d17111057 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=96=87=E8=89=B3?= Date: Thu, 8 Apr 2021 17:54:07 +0800 Subject: [PATCH] =?UTF-8?q?=E5=8D=87=E7=BA=A7Es=E9=A9=B1=E5=8A=A8=E7=89=88?= =?UTF-8?q?=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dal/seawater/dao/SeawaterDaoImpl.java | 2 +- .../script/jusfoun/JusfounCommon.groovy | 6 +- .../jusfoun/RentHouseExportBatch.groovy | 8 - .../sync/AnalysisOpenUriLogSyncBatch.groovy | 11 +- .../fangjia/script/util/ElasticUtils.groovy | 330 +++++++++--------- .../script/util/ElasticsearchHelper.groovy | 139 -------- .../RemoveInvalidEsfHouseSample.groovy | 5 +- .../script/wenyan/temp/xiaoqugeo.groovy | 19 +- 8 files changed, 199 insertions(+), 321 deletions(-) delete mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticsearchHelper.groovy diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java index 6df7e4df5..d2dccb33a 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java @@ -319,7 +319,7 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl i headers.put("Content-Type", "application/json"); headers.put("Authorization", "Basic "+authenticationEncoding+""); try { - String response = request.postWithBody("http://es.fangjia.com:9200/"+table+"/_search", reqBody, headers, "UTF-8"); + String response = request.postWithBody("http://es.fangjia.com:9200/"+table+"/_search?rest_total_hits_as_int=true", reqBody, headers, "UTF-8"); JSONObject json = JSONObject.fromObject(response); long time2 = System.currentTimeMillis(); System.out.println(time2-time1); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jusfoun/JusfounCommon.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jusfoun/JusfounCommon.groovy index fbbd9b746..494f8cb83 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jusfoun/JusfounCommon.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jusfoun/JusfounCommon.groovy @@ -4,7 +4,7 @@ import com.alibaba.fastjson.JSONObject import com.fangjia.core.util.DataUtil import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DbUtils -import com.fangjia.script.util.ElasticsearchHelper +import com.fangjia.script.util.ElasticUtils import com.mongodb.BasicDBObject import com.mongodb.DBObject import groovy.sql.GroovyRowResult @@ -622,13 +622,13 @@ class JusfounCommon { String scrollID = null) { try { if (transportClient == null) { - transportClient = ElasticsearchHelper.getClient("192.168.0.206"); + transportClient = ElasticUtils.getClient("192.168.0.206"); } return getEsDataOnce(transportClient, city, cat, startDate, endDate, INDEX, scrollID); } catch (Exception e) { e.printStackTrace() try { - transportClient = ElasticsearchHelper.getClient("192.168.0.206"); + transportClient = ElasticUtils.getClient("192.168.0.206"); return getEsDataOnce(transportClient, city, cat, startDate, endDate, INDEX, scrollID); } catch (Exception e1) { e1.printStackTrace() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jusfoun/RentHouseExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jusfoun/RentHouseExportBatch.groovy index bfb6410d3..8d8f994ed 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jusfoun/RentHouseExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jusfoun/RentHouseExportBatch.groovy @@ -8,21 +8,13 @@ import com.fangjia.script.Watcher import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ElasticUtils -import com.fangjia.script.util.ElasticsearchHelper -import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject -import com.mongodb.Bytes -import com.mongodb.DBCollection import com.mongodb.DBObject import org.apache.commons.lang.StringUtils -import org.elasticsearch.client.transport.TransportClient import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilders import java.sql.BatchUpdateException -import java.util.concurrent.LinkedBlockingDeque -import java.util.concurrent.ThreadPoolExecutor -import java.util.concurrent.TimeUnit @BatchOwner(Watcher.WangWenYan) class RentHouseExportBatch extends BaseBatch { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/sync/AnalysisOpenUriLogSyncBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/sync/AnalysisOpenUriLogSyncBatch.groovy index b5849e738..a634ea6dc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/sync/AnalysisOpenUriLogSyncBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/sync/AnalysisOpenUriLogSyncBatch.groovy @@ -5,7 +5,7 @@ import com.fangjia.core.util.DateUtil import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher -import com.fangjia.script.util.ElasticsearchHelper +import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection @@ -53,11 +53,8 @@ class AnalysisOpenUriLogSyncBatch extends BaseBatch { void process(Object args) { Map userInfo = getUserInfo() -// def startDate = DateUtil.truncateDay(DateUtil.add(new Date(), -3)) -// def endDate = DateUtil.truncateDay(new Date()) - def startDate = DateUtil.truncateDay(Date.parse("yyyyMMdd", "20210308")) - def endDate = DateUtil.truncateDay(Date.parse("yyyyMMdd", "20210314")) - + def startDate = DateUtil.truncateDay(DateUtil.add(new Date(), -3)) + def endDate = DateUtil.truncateDay(new Date()) BasicDBObject query = new BasicDBObject() query.append("c_date", new BasicDBObject('$gte', startDate).append('$lt', endDate)) @@ -66,7 +63,7 @@ class AnalysisOpenUriLogSyncBatch extends BaseBatch { long count = 0 long total = cursor.count() print("total="+total) - BulkProcessor bulkProcessor = ElasticsearchHelper.createBulkProcessor("seawater.es.db.fangjia.com") + BulkProcessor bulkProcessor = ElasticUtils.createBulkProcessor("seawater.es.db.fangjia.com") for(BasicDBObject log : cursor){ Map parameter_map = (Map) log.get("parameter_map") if(parameter_map == null) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy index 1f4442168..62c388a22 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy @@ -1,122 +1,170 @@ package com.fangjia.script.util -import org.apache.commons.lang3.SystemUtils +import org.apache.http.HttpHost +import org.apache.http.auth.AuthScope +import org.apache.http.auth.UsernamePasswordCredentials +import org.apache.http.client.CredentialsProvider +import org.apache.http.client.config.RequestConfig +import org.apache.http.impl.client.BasicCredentialsProvider +import org.apache.http.impl.nio.client.HttpAsyncClientBuilder +import org.elasticsearch.action.ActionListener import org.elasticsearch.action.bulk.BackoffPolicy import org.elasticsearch.action.bulk.BulkProcessor import org.elasticsearch.action.bulk.BulkRequest import org.elasticsearch.action.bulk.BulkResponse import org.elasticsearch.action.index.IndexRequest import org.elasticsearch.action.search.ClearScrollRequest +import org.elasticsearch.action.search.SearchRequest import org.elasticsearch.action.search.SearchResponse -import org.elasticsearch.action.search.SearchType -import org.elasticsearch.client.transport.TransportClient -import org.elasticsearch.common.settings.Settings -import org.elasticsearch.common.transport.TransportAddress -import org.elasticsearch.common.unit.ByteSizeUnit -import org.elasticsearch.common.unit.ByteSizeValue +import org.elasticsearch.action.search.SearchScrollRequest +import org.elasticsearch.client.RequestOptions +import org.elasticsearch.client.RestClient +import org.elasticsearch.client.RestClientBuilder +import org.elasticsearch.client.RestHighLevelClient import org.elasticsearch.common.unit.TimeValue import org.elasticsearch.index.query.QueryBuilder -import org.elasticsearch.index.query.QueryBuilders +import org.elasticsearch.search.Scroll import org.elasticsearch.search.SearchHit -import org.elasticsearch.search.sort.FieldSortBuilder -import org.elasticsearch.search.sort.SortOrder -import org.elasticsearch.xpack.client.PreBuiltXPackTransportClient +import org.elasticsearch.search.builder.SearchSourceBuilder +import org.elasticsearch.xpack.sql.jdbc.EsDataSource +import java.sql.Connection +import java.sql.ResultSet +import java.sql.Statement import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.TimeUnit +import java.util.function.BiConsumer +/** + * 功能说明:Elasticsearch工具类 + */ class ElasticUtils { - private static TransportClient client = null; - public static TransportClient getClient(def host){ - if(client != null){ - return client; + static getEsDataSource(String host){ + EsDataSource dataSource = new EsDataSource (); + dataSource.setUrl("jdbc:es://${host}:9200?timezone=UTC&page.size=240&connect.timeout=240&network.timeout=600&page.timeout=240"); + Properties connectionProperties = new Properties(); + connectionProperties.setProperty("user","developer"); + connectionProperties.setProperty("password","5e8edd851d2fdfbd7415232c67367cc3"); + dataSource.setProperties(connectionProperties); + return dataSource; + } + + static executeSqlQuery(String sql, Closure closure, String host = "192.168.0.216"){ + EsDataSource dataSource = getEsDataSource(host) + Connection connection = dataSource.getConnection() + Statement statement = connection.createStatement() + ResultSet rs = statement.executeQuery(sql) + def columns = [] + for(int i = 1; i <= rs.getMetaData().getColumnCount(); i++){ + columns << rs.getMetaData().getColumnName(i) } - - def certPath = "" - if(SystemUtils.IS_OS_WINDOWS){ - certPath= "D:\\goojiaconfig\\seawater\\conf\\elastic-certificates.p12" - }else{ - certPath = "/opt/elasticmongo/elastic-certificates.p12" + rs.getMetaData().getColumnCount() + while(rs.next()){ + def row = [:] + for (String col : columns){ + row[col] = rs.getObject(col) + } + closure.call(row) } - Settings.Builder builder = Settings.builder() - .put("cluster.name", "fangjia") - .put("client.transport.sniff", true) - - builder.put("xpack.security.user", "developer" + ":" + "5e8edd851d2fdfbd7415232c67367cc3") - .put("xpack.security.transport.ssl.enabled", "true") - .put("xpack.security.transport.ssl.truststore.path", certPath) - .put("xpack.security.transport.ssl.keystore.path", certPath) - .put("xpack.security.transport.ssl.verification_mode", "certificate") - - Settings settings = builder.build(); - client = new PreBuiltXPackTransportClient(settings) - .addTransportAddress(new TransportAddress(InetAddress.getByName(host), 9300)) - return client + rs.close() + connection.close() } - static executeQuery(String host, String indexName, QueryBuilder qb, Closure closure, String type= "_doc"){ - - client = getClient(host); - - BulkProcessor bulkProcessor = BulkProcessor.builder( - client, - new BulkProcessor.Listener() { - @Override - void beforeBulk(long executionId, - BulkRequest request) { - } - - @Override - void afterBulk(long executionId, - BulkRequest request, - BulkResponse response) { - } - - @Override - void afterBulk(long executionId, - BulkRequest request, - Throwable failure) { - } - }) - .setBulkActions(10000) - .setBulkSize(new ByteSizeValue(5, ByteSizeUnit.MB)) - .setFlushInterval(TimeValue.timeValueSeconds(5)) - .setConcurrentRequests(20) - .setBackoffPolicy(BackoffPolicy.exponentialBackoff(TimeValue.timeValueMillis(100), 10)) - .build() - - SearchResponse scrollResp = null; - int errorCount = 0; - def hits ; - def stop = false; - while(!stop){ - try{ - scrollResp = client.prepareSearch(indexName) - .addSort(FieldSortBuilder.DOC_FIELD_NAME, SortOrder.ASC) - .setScroll(new TimeValue(600000)) - .setQuery(qb) - .setSize(1000).get() - hits = scrollResp.getHits().getHits() - stop = true; - }catch(Exception e){ - errorCount++; - if(errorCount>11){ - stop = true; - } - println "错误:${errorCount} "+e.getMessage() + static RestHighLevelClient getClient(String host) { + RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http")); + builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() { + @Override + RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) { + requestConfigBuilder.setConnectionRequestTimeout(1000 * 60); + requestConfigBuilder.setSocketTimeout(1000 * 60); + return requestConfigBuilder; } + }) + builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() { + @Override + HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) { + httpAsyncClientBuilder.disableAuthCaching(); + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "goojia123456")); + httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider); + return httpAsyncClientBuilder; + } + }) + return new RestHighLevelClient(builder) + } + + + /** + * 创建一个ES批处理器 + * @param host + * @return + */ + static createBulkProcessor(String host, RestHighLevelClient client = null) { + + BulkProcessor.Listener listener = new BulkProcessor.Listener() { + @Override + void beforeBulk(long executionId, BulkRequest request) { + } + + @Override + void afterBulk(long executionId, BulkRequest request, + BulkResponse response) { + } + + @Override + void afterBulk(long executionId, BulkRequest request, + Throwable failure) { + } + }; + if(!client){ + client = getClient(host); } + return BulkProcessor.builder(new BiConsumer>() { + @Override + void accept(BulkRequest bulkRequest, ActionListener bulkResponseActionListener) { + client.bulkAsync(bulkRequest, RequestOptions.DEFAULT, bulkResponseActionListener) + } + }, listener).setBulkActions(10000) + .setConcurrentRequests(20) + .setBackoffPolicy(BackoffPolicy.constantBackoff(TimeValue.timeValueSeconds(1L), 3)) + .setFlushInterval(TimeValue.timeValueSeconds(10L)).build(); + } - if(hits){ - def nThread = Runtime.getRuntime().availableProcessors() * 5 - ExecutorService executorService = Executors.newFixedThreadPool(nThread) + /** + * 关闭一个ES批处理器 + * @param + * @return + */ + static closeBulkProcessor(BulkProcessor bulkProcessor) { - for (int k=0;k 0) { + for (int k=0; k < searchHits.size(); k++) { + final SearchHit hit = searchHits[k]; executorService.submit(new Runnable() { @Override void run() { @@ -124,96 +172,66 @@ class ElasticUtils { if(closure.getMaximumNumberOfParameters() == 1){ def update = closure.call(hit.getSourceAsMap()) if(update){ - bulkProcessor.add(new IndexRequest(indexName, type, hit.getId()).source(hit.getSourceAsMap())) + bulkProcessor.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap())) } }else if(closure.getMaximumNumberOfParameters() == 2){ def update = closure.call(hit.getId(), hit.getSourceAsMap()) if(update){ - bulkProcessor.add(new IndexRequest(indexName, type, hit.getId()).source(hit.getSourceAsMap())) + bulkProcessor.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap())) } } }catch (Exception e){ - // println(hits) e.printStackTrace() } } }) } - scrollResp = client.prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(60000)).execute().actionGet() - while(scrollResp.getHits().getHits().length != 0){ - hits = scrollResp.getHits().getHits(); - for (int k=0;k11){ - stop = true; - } - println "错误:${errorCount} "+e.getMessage() - } - } + final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L)) + RestHighLevelClient client = getClient(host) + SearchRequest searchRequest = new SearchRequest(indexName); + searchRequest.scroll(scroll); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(qb); + searchRequest.source(searchSourceBuilder); + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + String scrollId = searchResponse.getScrollId(); + SearchHit[] searchHits = searchResponse.getHits().getHits(); def results = [] - while(scrollResp.getHits() && scrollResp.getHits().getHits().length != 0){ - for (SearchHit searchHit : scrollResp.getHits().getHits()) { + while (searchHits != null && searchHits.length > 0) { + for (int k=0; k < searchHits.size(); k++) { + SearchHit searchHit = searchHits[k] searchHit.getSourceAsMap().put("id", searchHit.getId()); results << searchHit.getSourceAsMap() } - scrollResp = client.prepareSearchScroll(scrollResp.getScrollId()).setScroll(TimeValue.timeValueMinutes(1)).execute().actionGet(); - //当searchHits的数组为空的时候结束循环,至此数据全部读取完毕 + SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); + scrollRequest.scroll(scroll); + searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT); + scrollId = searchResponse.getScrollId(); + searchHits = searchResponse.getHits().getHits(); } - //删除scroll ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); - clearScrollRequest.addScrollId(scrollResp.getScrollId()); - client.clearScroll(clearScrollRequest).actionGet(); + clearScrollRequest.addScrollId(scrollId); + client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT); + client.close() return results; } - } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticsearchHelper.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticsearchHelper.groovy deleted file mode 100644 index 7cbf5c37b..000000000 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticsearchHelper.groovy +++ /dev/null @@ -1,139 +0,0 @@ -package com.fangjia.script.util - - -import org.apache.http.HttpHost -import org.apache.http.auth.AuthScope -import org.apache.http.auth.UsernamePasswordCredentials -import org.apache.http.client.CredentialsProvider -import org.apache.http.client.config.RequestConfig -import org.apache.http.impl.client.BasicCredentialsProvider -import org.apache.http.impl.nio.client.HttpAsyncClientBuilder -import org.elasticsearch.action.ActionListener -import org.elasticsearch.action.bulk.BackoffPolicy -import org.elasticsearch.action.bulk.BulkProcessor -import org.elasticsearch.action.bulk.BulkRequest -import org.elasticsearch.action.bulk.BulkResponse -import org.elasticsearch.client.RestClient -import org.elasticsearch.client.RestClientBuilder -import org.elasticsearch.client.RestHighLevelClient -import org.elasticsearch.client.RequestOptions -import org.elasticsearch.common.unit.TimeValue -import org.elasticsearch.xpack.sql.jdbc.EsDataSource - -import java.sql.Connection -import java.sql.ResultSet -import java.sql.Statement -import java.util.concurrent.TimeUnit -import java.util.function.BiConsumer - -/** - * 功能说明:Elasticsearch工具类 - */ -class ElasticsearchHelper { - - - static getEsDataSource(String host){ - EsDataSource dataSource = new EsDataSource (); - dataSource.setUrl("jdbc:es://${host}:9200?timezone=UTC&page.size=240&connect.timeout=240&network.timeout=600&page.timeout=240"); - Properties connectionProperties = new Properties(); - connectionProperties.setProperty("user","developer"); - connectionProperties.setProperty("password","5e8edd851d2fdfbd7415232c67367cc3"); - dataSource.setProperties(connectionProperties); - return dataSource; - } - - static executeQuery(String sql, Closure closure, String host = "192.168.0.216"){ - EsDataSource dataSource = getEsDataSource(host) - Connection connection = dataSource.getConnection() - Statement statement = connection.createStatement() - ResultSet rs = statement.executeQuery(sql) - def columns = [] - for(int i = 1; i <= rs.getMetaData().getColumnCount(); i++){ - columns << rs.getMetaData().getColumnName(i) - } - rs.getMetaData().getColumnCount() - while(rs.next()){ - def row = [:] - for (String col : columns){ - row[col] = rs.getObject(col) - } - closure.call(row) - } - rs.close() - connection.close() - } - - - static RestHighLevelClient getClient(String host) { - RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http")); - builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() { - @Override - RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) { - requestConfigBuilder.setConnectionRequestTimeout(1000 * 60); - requestConfigBuilder.setSocketTimeout(1000 * 60); - return requestConfigBuilder; - } - }) - builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() { - @Override - HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) { - httpAsyncClientBuilder.disableAuthCaching(); - CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "goojia123456")); - httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider); - return httpAsyncClientBuilder; - } - }) - return new RestHighLevelClient(builder) - } - - - /** - * 创建一个ES批处理器 - * @param host - * @return - */ - static createBulkProcessor(String host) { - BulkProcessor.Listener listener = new BulkProcessor.Listener() { - @Override - void beforeBulk(long executionId, BulkRequest request) { - } - - @Override - void afterBulk(long executionId, BulkRequest request, - BulkResponse response) { - } - - @Override - void afterBulk(long executionId, BulkRequest request, - Throwable failure) { - } - }; - RestHighLevelClient client = getClient(host); - - return BulkProcessor.builder(new BiConsumer>() { - @Override - void accept(BulkRequest bulkRequest, ActionListener bulkResponseActionListener) { - client.bulkAsync(bulkRequest, RequestOptions.DEFAULT, bulkResponseActionListener) - } - }, listener).setBulkActions(10000) - .setConcurrentRequests(20) - .setBackoffPolicy(BackoffPolicy.constantBackoff(TimeValue.timeValueSeconds(1L), 3)) - .setFlushInterval(TimeValue.timeValueSeconds(10L)).build(); - } - - /** - * 关闭一个ES批处理器 - * @param - * @return - */ - static closeBulkProcessor(BulkProcessor bulkProcessor) { - - bulkProcessor.awaitClose(10, TimeUnit.DAYS) - bulkProcessor.flush() - bulkProcessor.close() - - } - - -} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/vipsite/RemoveInvalidEsfHouseSample.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/vipsite/RemoveInvalidEsfHouseSample.groovy index a9a6436e8..c4eabb0fc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/vipsite/RemoveInvalidEsfHouseSample.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/vipsite/RemoveInvalidEsfHouseSample.groovy @@ -1,19 +1,18 @@ package com.fangjia.script.vipsite -import com.fangjia.script.util.ElasticsearchHelper +import com.fangjia.script.util.ElasticUtils import com.mongodb.DBObject import org.elasticsearch.action.search.SearchRequestBuilder import org.elasticsearch.action.search.SearchResponse import org.elasticsearch.action.search.SearchType import org.elasticsearch.client.transport.TransportClient -import org.elasticsearch.common.unit.TimeValue import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilders import org.elasticsearch.search.SearchHit import org.elasticsearch.search.SearchHits try { - TransportClient transportClient = ElasticsearchHelper.getClient("192.168.0.206"); + TransportClient transportClient = ElasticUtils.getClient("192.168.0.206"); QueryBuilder dateRange_avg_price = QueryBuilders.rangeQuery("avg_price").lte(0); QueryBuilder dateRange_total_price = QueryBuilders.rangeQuery("total_price").lte(0); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/xiaoqugeo.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/xiaoqugeo.groovy index eb226eba3..be686fa81 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/xiaoqugeo.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/xiaoqugeo.groovy @@ -1,7 +1,18 @@ package com.fangjia.script.wenyan.temp -import com.fangjia.script.util.ElasticsearchHelper +import com.fangjia.script.util.ElasticUtils +import org.elasticsearch.index.query.QueryBuilders + +//ElasticUtils.executeSqlQuery("select * from seawater where p_date > 20210407 limit 10",{ Map row-> +// println(row.name) +//}, "192.168.0.216") + + +//ElasticUtils.executeSimpleQuery("192.168.0.216", "seawater-2021", QueryBuilders.termQuery("district_name", "新泾三村")).each { +// println(it) +//} + +ElasticUtils.executeQuery("192.168.0.216", "seawater-2021", QueryBuilders.termQuery("district_name", "新泾三村"),{ + println(it) +}) -ElasticsearchHelper.executeQuery("select * from seawater where p_date > 20210407 limit 10",{Map row-> - println(row.name) -}, "192.168.0.216") \ No newline at end of file