升级Es驱动版本
This commit is contained in:
+1
-1
@@ -319,7 +319,7 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl<Seawater> i
|
||||
headers.put("Content-Type", "application/json");
|
||||
headers.put("Authorization", "Basic "+authenticationEncoding+"");
|
||||
try {
|
||||
String response = request.postWithBody("http://es.fangjia.com:9200/"+table+"/_search", reqBody, headers, "UTF-8");
|
||||
String response = request.postWithBody("http://es.fangjia.com:9200/"+table+"/_search?rest_total_hits_as_int=true", reqBody, headers, "UTF-8");
|
||||
JSONObject json = JSONObject.fromObject(response);
|
||||
long time2 = System.currentTimeMillis();
|
||||
System.out.println(time2-time1);
|
||||
|
||||
+3
-3
@@ -4,7 +4,7 @@ import com.alibaba.fastjson.JSONObject
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticsearchHelper
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBObject
|
||||
import groovy.sql.GroovyRowResult
|
||||
@@ -622,13 +622,13 @@ class JusfounCommon {
|
||||
String scrollID = null) {
|
||||
try {
|
||||
if (transportClient == null) {
|
||||
transportClient = ElasticsearchHelper.getClient("192.168.0.206");
|
||||
transportClient = ElasticUtils.getClient("192.168.0.206");
|
||||
}
|
||||
return getEsDataOnce(transportClient, city, cat, startDate, endDate, INDEX, scrollID);
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
try {
|
||||
transportClient = ElasticsearchHelper.getClient("192.168.0.206");
|
||||
transportClient = ElasticUtils.getClient("192.168.0.206");
|
||||
return getEsDataOnce(transportClient, city, cat, startDate, endDate, INDEX, scrollID);
|
||||
} catch (Exception e1) {
|
||||
e1.printStackTrace()
|
||||
|
||||
-8
@@ -8,21 +8,13 @@ import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ElasticsearchHelper
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.client.transport.TransportClient
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.sql.BatchUpdateException
|
||||
import java.util.concurrent.LinkedBlockingDeque
|
||||
import java.util.concurrent.ThreadPoolExecutor
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
@BatchOwner(Watcher.WangWenYan)
|
||||
class RentHouseExportBatch extends BaseBatch {
|
||||
|
||||
+4
-7
@@ -5,7 +5,7 @@ import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.util.ElasticsearchHelper
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
@@ -53,11 +53,8 @@ class AnalysisOpenUriLogSyncBatch extends BaseBatch {
|
||||
void process(Object args) {
|
||||
|
||||
Map userInfo = getUserInfo()
|
||||
// def startDate = DateUtil.truncateDay(DateUtil.add(new Date(), -3))
|
||||
// def endDate = DateUtil.truncateDay(new Date())
|
||||
def startDate = DateUtil.truncateDay(Date.parse("yyyyMMdd", "20210308"))
|
||||
def endDate = DateUtil.truncateDay(Date.parse("yyyyMMdd", "20210314"))
|
||||
|
||||
def startDate = DateUtil.truncateDay(DateUtil.add(new Date(), -3))
|
||||
def endDate = DateUtil.truncateDay(new Date())
|
||||
BasicDBObject query = new BasicDBObject()
|
||||
query.append("c_date", new BasicDBObject('$gte', startDate).append('$lt', endDate))
|
||||
|
||||
@@ -66,7 +63,7 @@ class AnalysisOpenUriLogSyncBatch extends BaseBatch {
|
||||
long count = 0
|
||||
long total = cursor.count()
|
||||
print("total="+total)
|
||||
BulkProcessor bulkProcessor = ElasticsearchHelper.createBulkProcessor("seawater.es.db.fangjia.com")
|
||||
BulkProcessor bulkProcessor = ElasticUtils.createBulkProcessor("seawater.es.db.fangjia.com")
|
||||
for(BasicDBObject log : cursor){
|
||||
Map<String,Object> parameter_map = (Map) log.get("parameter_map")
|
||||
if(parameter_map == null) {
|
||||
|
||||
+174
-156
@@ -1,122 +1,170 @@
|
||||
package com.fangjia.script.util
|
||||
|
||||
import org.apache.commons.lang3.SystemUtils
|
||||
import org.apache.http.HttpHost
|
||||
import org.apache.http.auth.AuthScope
|
||||
import org.apache.http.auth.UsernamePasswordCredentials
|
||||
import org.apache.http.client.CredentialsProvider
|
||||
import org.apache.http.client.config.RequestConfig
|
||||
import org.apache.http.impl.client.BasicCredentialsProvider
|
||||
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder
|
||||
import org.elasticsearch.action.ActionListener
|
||||
import org.elasticsearch.action.bulk.BackoffPolicy
|
||||
import org.elasticsearch.action.bulk.BulkProcessor
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
import org.elasticsearch.action.index.IndexRequest
|
||||
import org.elasticsearch.action.search.ClearScrollRequest
|
||||
import org.elasticsearch.action.search.SearchRequest
|
||||
import org.elasticsearch.action.search.SearchResponse
|
||||
import org.elasticsearch.action.search.SearchType
|
||||
import org.elasticsearch.client.transport.TransportClient
|
||||
import org.elasticsearch.common.settings.Settings
|
||||
import org.elasticsearch.common.transport.TransportAddress
|
||||
import org.elasticsearch.common.unit.ByteSizeUnit
|
||||
import org.elasticsearch.common.unit.ByteSizeValue
|
||||
import org.elasticsearch.action.search.SearchScrollRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.client.RestClient
|
||||
import org.elasticsearch.client.RestClientBuilder
|
||||
import org.elasticsearch.client.RestHighLevelClient
|
||||
import org.elasticsearch.common.unit.TimeValue
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.search.Scroll
|
||||
import org.elasticsearch.search.SearchHit
|
||||
import org.elasticsearch.search.sort.FieldSortBuilder
|
||||
import org.elasticsearch.search.sort.SortOrder
|
||||
import org.elasticsearch.xpack.client.PreBuiltXPackTransportClient
|
||||
import org.elasticsearch.search.builder.SearchSourceBuilder
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.TimeUnit
|
||||
import java.util.function.BiConsumer
|
||||
|
||||
/**
|
||||
* 功能说明:Elasticsearch工具类
|
||||
*/
|
||||
class ElasticUtils {
|
||||
|
||||
private static TransportClient client = null;
|
||||
|
||||
public static TransportClient getClient(def host){
|
||||
if(client != null){
|
||||
return client;
|
||||
static getEsDataSource(String host){
|
||||
EsDataSource dataSource = new EsDataSource ();
|
||||
dataSource.setUrl("jdbc:es://${host}:9200?timezone=UTC&page.size=240&connect.timeout=240&network.timeout=600&page.timeout=240");
|
||||
Properties connectionProperties = new Properties();
|
||||
connectionProperties.setProperty("user","developer");
|
||||
connectionProperties.setProperty("password","5e8edd851d2fdfbd7415232c67367cc3");
|
||||
dataSource.setProperties(connectionProperties);
|
||||
return dataSource;
|
||||
}
|
||||
|
||||
static executeSqlQuery(String sql, Closure<Map> closure, String host = "192.168.0.216"){
|
||||
EsDataSource dataSource = getEsDataSource(host)
|
||||
Connection connection = dataSource.getConnection()
|
||||
Statement statement = connection.createStatement()
|
||||
ResultSet rs = statement.executeQuery(sql)
|
||||
def columns = []
|
||||
for(int i = 1; i <= rs.getMetaData().getColumnCount(); i++){
|
||||
columns << rs.getMetaData().getColumnName(i)
|
||||
}
|
||||
|
||||
def certPath = ""
|
||||
if(SystemUtils.IS_OS_WINDOWS){
|
||||
certPath= "D:\\goojiaconfig\\seawater\\conf\\elastic-certificates.p12"
|
||||
}else{
|
||||
certPath = "/opt/elasticmongo/elastic-certificates.p12"
|
||||
rs.getMetaData().getColumnCount()
|
||||
while(rs.next()){
|
||||
def row = [:]
|
||||
for (String col : columns){
|
||||
row[col] = rs.getObject(col)
|
||||
}
|
||||
closure.call(row)
|
||||
}
|
||||
Settings.Builder builder = Settings.builder()
|
||||
.put("cluster.name", "fangjia")
|
||||
.put("client.transport.sniff", true)
|
||||
|
||||
builder.put("xpack.security.user", "developer" + ":" + "5e8edd851d2fdfbd7415232c67367cc3")
|
||||
.put("xpack.security.transport.ssl.enabled", "true")
|
||||
.put("xpack.security.transport.ssl.truststore.path", certPath)
|
||||
.put("xpack.security.transport.ssl.keystore.path", certPath)
|
||||
.put("xpack.security.transport.ssl.verification_mode", "certificate")
|
||||
|
||||
Settings settings = builder.build();
|
||||
client = new PreBuiltXPackTransportClient(settings)
|
||||
.addTransportAddress(new TransportAddress(InetAddress.getByName(host), 9300))
|
||||
return client
|
||||
rs.close()
|
||||
connection.close()
|
||||
}
|
||||
|
||||
|
||||
static executeQuery(String host, String indexName, QueryBuilder qb, Closure closure, String type= "_doc"){
|
||||
|
||||
client = getClient(host);
|
||||
|
||||
BulkProcessor bulkProcessor = BulkProcessor.builder(
|
||||
client,
|
||||
new BulkProcessor.Listener() {
|
||||
@Override
|
||||
void beforeBulk(long executionId,
|
||||
BulkRequest request) {
|
||||
}
|
||||
|
||||
@Override
|
||||
void afterBulk(long executionId,
|
||||
BulkRequest request,
|
||||
BulkResponse response) {
|
||||
}
|
||||
|
||||
@Override
|
||||
void afterBulk(long executionId,
|
||||
BulkRequest request,
|
||||
Throwable failure) {
|
||||
}
|
||||
})
|
||||
.setBulkActions(10000)
|
||||
.setBulkSize(new ByteSizeValue(5, ByteSizeUnit.MB))
|
||||
.setFlushInterval(TimeValue.timeValueSeconds(5))
|
||||
.setConcurrentRequests(20)
|
||||
.setBackoffPolicy(BackoffPolicy.exponentialBackoff(TimeValue.timeValueMillis(100), 10))
|
||||
.build()
|
||||
|
||||
SearchResponse scrollResp = null;
|
||||
int errorCount = 0;
|
||||
def hits ;
|
||||
def stop = false;
|
||||
while(!stop){
|
||||
try{
|
||||
scrollResp = client.prepareSearch(indexName)
|
||||
.addSort(FieldSortBuilder.DOC_FIELD_NAME, SortOrder.ASC)
|
||||
.setScroll(new TimeValue(600000))
|
||||
.setQuery(qb)
|
||||
.setSize(1000).get()
|
||||
hits = scrollResp.getHits().getHits()
|
||||
stop = true;
|
||||
}catch(Exception e){
|
||||
errorCount++;
|
||||
if(errorCount>11){
|
||||
stop = true;
|
||||
}
|
||||
println "错误:${errorCount} "+e.getMessage()
|
||||
static RestHighLevelClient getClient(String host) {
|
||||
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
|
||||
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
|
||||
@Override
|
||||
RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
|
||||
requestConfigBuilder.setConnectionRequestTimeout(1000 * 60);
|
||||
requestConfigBuilder.setSocketTimeout(1000 * 60);
|
||||
return requestConfigBuilder;
|
||||
}
|
||||
})
|
||||
builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
|
||||
@Override
|
||||
HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
|
||||
httpAsyncClientBuilder.disableAuthCaching();
|
||||
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
|
||||
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "goojia123456"));
|
||||
httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
|
||||
return httpAsyncClientBuilder;
|
||||
}
|
||||
})
|
||||
return new RestHighLevelClient(builder)
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 创建一个ES批处理器
|
||||
* @param host
|
||||
* @return
|
||||
*/
|
||||
static createBulkProcessor(String host, RestHighLevelClient client = null) {
|
||||
|
||||
BulkProcessor.Listener listener = new BulkProcessor.Listener() {
|
||||
@Override
|
||||
void beforeBulk(long executionId, BulkRequest request) {
|
||||
}
|
||||
|
||||
@Override
|
||||
void afterBulk(long executionId, BulkRequest request,
|
||||
BulkResponse response) {
|
||||
}
|
||||
|
||||
@Override
|
||||
void afterBulk(long executionId, BulkRequest request,
|
||||
Throwable failure) {
|
||||
}
|
||||
};
|
||||
if(!client){
|
||||
client = getClient(host);
|
||||
}
|
||||
return BulkProcessor.builder(new BiConsumer<BulkRequest, ActionListener<BulkResponse>>() {
|
||||
@Override
|
||||
void accept(BulkRequest bulkRequest, ActionListener<BulkResponse> bulkResponseActionListener) {
|
||||
client.bulkAsync(bulkRequest, RequestOptions.DEFAULT, bulkResponseActionListener)
|
||||
}
|
||||
}, listener).setBulkActions(10000)
|
||||
.setConcurrentRequests(20)
|
||||
.setBackoffPolicy(BackoffPolicy.constantBackoff(TimeValue.timeValueSeconds(1L), 3))
|
||||
.setFlushInterval(TimeValue.timeValueSeconds(10L)).build();
|
||||
}
|
||||
|
||||
if(hits){
|
||||
def nThread = Runtime.getRuntime().availableProcessors() * 5
|
||||
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
|
||||
/**
|
||||
* 关闭一个ES批处理器
|
||||
* @param
|
||||
* @return
|
||||
*/
|
||||
static closeBulkProcessor(BulkProcessor bulkProcessor) {
|
||||
|
||||
for (int k=0;k<hits.size();k++) {
|
||||
final SearchHit hit = hits[k];
|
||||
bulkProcessor.awaitClose(10, TimeUnit.DAYS)
|
||||
bulkProcessor.flush()
|
||||
bulkProcessor.close()
|
||||
|
||||
}
|
||||
|
||||
static executeQuery(String host, String indexName, QueryBuilder qb, Closure closure){
|
||||
|
||||
RestHighLevelClient client = getClient(host)
|
||||
BulkProcessor bulkProcessor = createBulkProcessor(host, client)
|
||||
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L))
|
||||
SearchRequest searchRequest = new SearchRequest(indexName);
|
||||
searchRequest.scroll(scroll);
|
||||
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
|
||||
searchSourceBuilder.query(qb);
|
||||
searchRequest.source(searchSourceBuilder);
|
||||
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
|
||||
String scrollId = searchResponse.getScrollId();
|
||||
SearchHit[] searchHits = searchResponse.getHits().getHits();
|
||||
def nThread = Runtime.getRuntime().availableProcessors() * 5
|
||||
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
|
||||
while (searchHits != null && searchHits.length > 0) {
|
||||
for (int k=0; k < searchHits.size(); k++) {
|
||||
final SearchHit hit = searchHits[k];
|
||||
executorService.submit(new Runnable() {
|
||||
@Override
|
||||
void run() {
|
||||
@@ -124,96 +172,66 @@ class ElasticUtils {
|
||||
if(closure.getMaximumNumberOfParameters() == 1){
|
||||
def update = closure.call(hit.getSourceAsMap())
|
||||
if(update){
|
||||
bulkProcessor.add(new IndexRequest(indexName, type, hit.getId()).source(hit.getSourceAsMap()))
|
||||
bulkProcessor.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap()))
|
||||
}
|
||||
}else if(closure.getMaximumNumberOfParameters() == 2){
|
||||
def update = closure.call(hit.getId(), hit.getSourceAsMap())
|
||||
if(update){
|
||||
bulkProcessor.add(new IndexRequest(indexName, type, hit.getId()).source(hit.getSourceAsMap()))
|
||||
bulkProcessor.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap()))
|
||||
}
|
||||
}
|
||||
}catch (Exception e){
|
||||
// println(hits)
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
scrollResp = client.prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(60000)).execute().actionGet()
|
||||
while(scrollResp.getHits().getHits().length != 0){
|
||||
hits = scrollResp.getHits().getHits();
|
||||
for (int k=0;k<hits.size();k++) {
|
||||
final SearchHit hit = hits[k];
|
||||
executorService.submit(new Runnable() {
|
||||
@Override
|
||||
void run() {
|
||||
try{
|
||||
if(closure.getMaximumNumberOfParameters() == 1){
|
||||
def update = closure.call(hit.getSourceAsMap())
|
||||
if(update){
|
||||
bulkProcessor.add(new IndexRequest(indexName, type, hit.getId()).source(hit.getSourceAsMap()))
|
||||
}
|
||||
}else if(closure.getMaximumNumberOfParameters() == 2){
|
||||
def update = closure.call(hit.getId(), hit.getSourceAsMap())
|
||||
if(update){
|
||||
bulkProcessor.add(new IndexRequest(indexName, type, hit.getId()).source(hit.getSourceAsMap()))
|
||||
}
|
||||
}
|
||||
}catch (Exception e){
|
||||
// println(hits)
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
scrollResp = client.prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(60000)).execute().actionGet()
|
||||
}
|
||||
executorService.shutdown()
|
||||
executorService.awaitTermination(1, TimeUnit.DAYS)
|
||||
bulkProcessor.awaitClose(1, TimeUnit.DAYS)
|
||||
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
|
||||
scrollRequest.scroll(scroll);
|
||||
searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
|
||||
scrollId = searchResponse.getScrollId();
|
||||
searchHits = searchResponse.getHits().getHits();
|
||||
}
|
||||
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
|
||||
clearScrollRequest.addScrollId(scrollId);
|
||||
client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
|
||||
executorService.shutdown()
|
||||
executorService.awaitTermination(1, TimeUnit.DAYS)
|
||||
bulkProcessor.close()
|
||||
bulkProcessor.awaitClose(1, TimeUnit.DAYS)
|
||||
client.close()
|
||||
}
|
||||
|
||||
|
||||
static def executeSimpleQuery(String host, String indexName, QueryBuilder qb){
|
||||
client = getClient(host);
|
||||
SearchResponse scrollResp = null;
|
||||
int errorCount = 0;
|
||||
def hits ;
|
||||
def stop = false;
|
||||
while(!stop){
|
||||
try{
|
||||
scrollResp = client.prepareSearch(indexName)
|
||||
.addSort(FieldSortBuilder.DOC_FIELD_NAME, SortOrder.ASC)
|
||||
.setScroll(new TimeValue(600000))
|
||||
.setQuery(qb)
|
||||
.setSize(10000)
|
||||
.setScroll(TimeValue.timeValueMinutes(1))//设置 search context 维护1分钟的有效期
|
||||
.get()
|
||||
|
||||
stop = true;
|
||||
}catch(Exception e){
|
||||
errorCount++;
|
||||
if(errorCount>11){
|
||||
stop = true;
|
||||
}
|
||||
println "错误:${errorCount} "+e.getMessage()
|
||||
}
|
||||
}
|
||||
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L))
|
||||
RestHighLevelClient client = getClient(host)
|
||||
SearchRequest searchRequest = new SearchRequest(indexName);
|
||||
searchRequest.scroll(scroll);
|
||||
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
|
||||
searchSourceBuilder.query(qb);
|
||||
searchRequest.source(searchSourceBuilder);
|
||||
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
|
||||
String scrollId = searchResponse.getScrollId();
|
||||
SearchHit[] searchHits = searchResponse.getHits().getHits();
|
||||
def results = []
|
||||
while(scrollResp.getHits() && scrollResp.getHits().getHits().length != 0){
|
||||
for (SearchHit searchHit : scrollResp.getHits().getHits()) {
|
||||
while (searchHits != null && searchHits.length > 0) {
|
||||
for (int k=0; k < searchHits.size(); k++) {
|
||||
SearchHit searchHit = searchHits[k]
|
||||
searchHit.getSourceAsMap().put("id", searchHit.getId());
|
||||
results << searchHit.getSourceAsMap()
|
||||
}
|
||||
scrollResp = client.prepareSearchScroll(scrollResp.getScrollId()).setScroll(TimeValue.timeValueMinutes(1)).execute().actionGet();
|
||||
//当searchHits的数组为空的时候结束循环,至此数据全部读取完毕
|
||||
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
|
||||
scrollRequest.scroll(scroll);
|
||||
searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
|
||||
scrollId = searchResponse.getScrollId();
|
||||
searchHits = searchResponse.getHits().getHits();
|
||||
}
|
||||
//删除scroll
|
||||
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
|
||||
clearScrollRequest.addScrollId(scrollResp.getScrollId());
|
||||
client.clearScroll(clearScrollRequest).actionGet();
|
||||
clearScrollRequest.addScrollId(scrollId);
|
||||
client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
|
||||
client.close()
|
||||
return results;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
-139
@@ -1,139 +0,0 @@
|
||||
package com.fangjia.script.util
|
||||
|
||||
|
||||
import org.apache.http.HttpHost
|
||||
import org.apache.http.auth.AuthScope
|
||||
import org.apache.http.auth.UsernamePasswordCredentials
|
||||
import org.apache.http.client.CredentialsProvider
|
||||
import org.apache.http.client.config.RequestConfig
|
||||
import org.apache.http.impl.client.BasicCredentialsProvider
|
||||
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder
|
||||
import org.elasticsearch.action.ActionListener
|
||||
import org.elasticsearch.action.bulk.BackoffPolicy
|
||||
import org.elasticsearch.action.bulk.BulkProcessor
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
import org.elasticsearch.client.RestClient
|
||||
import org.elasticsearch.client.RestClientBuilder
|
||||
import org.elasticsearch.client.RestHighLevelClient
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.common.unit.TimeValue
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
import java.util.concurrent.TimeUnit
|
||||
import java.util.function.BiConsumer
|
||||
|
||||
/**
|
||||
* 功能说明:Elasticsearch工具类
|
||||
*/
|
||||
class ElasticsearchHelper {
|
||||
|
||||
|
||||
static getEsDataSource(String host){
|
||||
EsDataSource dataSource = new EsDataSource ();
|
||||
dataSource.setUrl("jdbc:es://${host}:9200?timezone=UTC&page.size=240&connect.timeout=240&network.timeout=600&page.timeout=240");
|
||||
Properties connectionProperties = new Properties();
|
||||
connectionProperties.setProperty("user","developer");
|
||||
connectionProperties.setProperty("password","5e8edd851d2fdfbd7415232c67367cc3");
|
||||
dataSource.setProperties(connectionProperties);
|
||||
return dataSource;
|
||||
}
|
||||
|
||||
static executeQuery(String sql, Closure<Map> closure, String host = "192.168.0.216"){
|
||||
EsDataSource dataSource = getEsDataSource(host)
|
||||
Connection connection = dataSource.getConnection()
|
||||
Statement statement = connection.createStatement()
|
||||
ResultSet rs = statement.executeQuery(sql)
|
||||
def columns = []
|
||||
for(int i = 1; i <= rs.getMetaData().getColumnCount(); i++){
|
||||
columns << rs.getMetaData().getColumnName(i)
|
||||
}
|
||||
rs.getMetaData().getColumnCount()
|
||||
while(rs.next()){
|
||||
def row = [:]
|
||||
for (String col : columns){
|
||||
row[col] = rs.getObject(col)
|
||||
}
|
||||
closure.call(row)
|
||||
}
|
||||
rs.close()
|
||||
connection.close()
|
||||
}
|
||||
|
||||
|
||||
static RestHighLevelClient getClient(String host) {
|
||||
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
|
||||
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
|
||||
@Override
|
||||
RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
|
||||
requestConfigBuilder.setConnectionRequestTimeout(1000 * 60);
|
||||
requestConfigBuilder.setSocketTimeout(1000 * 60);
|
||||
return requestConfigBuilder;
|
||||
}
|
||||
})
|
||||
builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
|
||||
@Override
|
||||
HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
|
||||
httpAsyncClientBuilder.disableAuthCaching();
|
||||
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
|
||||
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "goojia123456"));
|
||||
httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
|
||||
return httpAsyncClientBuilder;
|
||||
}
|
||||
})
|
||||
return new RestHighLevelClient(builder)
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 创建一个ES批处理器
|
||||
* @param host
|
||||
* @return
|
||||
*/
|
||||
static createBulkProcessor(String host) {
|
||||
BulkProcessor.Listener listener = new BulkProcessor.Listener() {
|
||||
@Override
|
||||
void beforeBulk(long executionId, BulkRequest request) {
|
||||
}
|
||||
|
||||
@Override
|
||||
void afterBulk(long executionId, BulkRequest request,
|
||||
BulkResponse response) {
|
||||
}
|
||||
|
||||
@Override
|
||||
void afterBulk(long executionId, BulkRequest request,
|
||||
Throwable failure) {
|
||||
}
|
||||
};
|
||||
RestHighLevelClient client = getClient(host);
|
||||
|
||||
return BulkProcessor.builder(new BiConsumer<BulkRequest, ActionListener<BulkResponse>>() {
|
||||
@Override
|
||||
void accept(BulkRequest bulkRequest, ActionListener<BulkResponse> bulkResponseActionListener) {
|
||||
client.bulkAsync(bulkRequest, RequestOptions.DEFAULT, bulkResponseActionListener)
|
||||
}
|
||||
}, listener).setBulkActions(10000)
|
||||
.setConcurrentRequests(20)
|
||||
.setBackoffPolicy(BackoffPolicy.constantBackoff(TimeValue.timeValueSeconds(1L), 3))
|
||||
.setFlushInterval(TimeValue.timeValueSeconds(10L)).build();
|
||||
}
|
||||
|
||||
/**
|
||||
* 关闭一个ES批处理器
|
||||
* @param
|
||||
* @return
|
||||
*/
|
||||
static closeBulkProcessor(BulkProcessor bulkProcessor) {
|
||||
|
||||
bulkProcessor.awaitClose(10, TimeUnit.DAYS)
|
||||
bulkProcessor.flush()
|
||||
bulkProcessor.close()
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
+2
-3
@@ -1,19 +1,18 @@
|
||||
package com.fangjia.script.vipsite
|
||||
|
||||
import com.fangjia.script.util.ElasticsearchHelper
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.mongodb.DBObject
|
||||
import org.elasticsearch.action.search.SearchRequestBuilder
|
||||
import org.elasticsearch.action.search.SearchResponse
|
||||
import org.elasticsearch.action.search.SearchType
|
||||
import org.elasticsearch.client.transport.TransportClient
|
||||
import org.elasticsearch.common.unit.TimeValue
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.search.SearchHit
|
||||
import org.elasticsearch.search.SearchHits
|
||||
|
||||
try {
|
||||
TransportClient transportClient = ElasticsearchHelper.getClient("192.168.0.206");
|
||||
TransportClient transportClient = ElasticUtils.getClient("192.168.0.206");
|
||||
|
||||
QueryBuilder dateRange_avg_price = QueryBuilders.rangeQuery("avg_price").lte(0);
|
||||
QueryBuilder dateRange_total_price = QueryBuilders.rangeQuery("total_price").lte(0);
|
||||
|
||||
+15
-4
@@ -1,7 +1,18 @@
|
||||
package com.fangjia.script.wenyan.temp
|
||||
|
||||
import com.fangjia.script.util.ElasticsearchHelper
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
//ElasticUtils.executeSqlQuery("select * from seawater where p_date > 20210407 limit 10",{ Map row->
|
||||
// println(row.name)
|
||||
//}, "192.168.0.216")
|
||||
|
||||
|
||||
//ElasticUtils.executeSimpleQuery("192.168.0.216", "seawater-2021", QueryBuilders.termQuery("district_name", "ÐÂãþÈý´å")).each {
|
||||
// println(it)
|
||||
//}
|
||||
|
||||
ElasticUtils.executeQuery("192.168.0.216", "seawater-2021", QueryBuilders.termQuery("district_name", "ÐÂãþÈý´å"),{
|
||||
println(it)
|
||||
})
|
||||
|
||||
ElasticsearchHelper.executeQuery("select * from seawater where p_date > 20210407 limit 10",{Map row->
|
||||
println(row.name)
|
||||
}, "192.168.0.216")
|
||||
Reference in New Issue
Block a user