+137
-6
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.script.gj.xiamen
|
||||
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import org.apache.http.HttpHost
|
||||
import org.apache.http.auth.AuthScope
|
||||
import org.apache.http.auth.UsernamePasswordCredentials
|
||||
@@ -7,8 +8,10 @@ import org.apache.http.client.CredentialsProvider
|
||||
import org.apache.http.client.config.RequestConfig
|
||||
import org.apache.http.impl.client.BasicCredentialsProvider
|
||||
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder
|
||||
import org.apache.solr.client.solrj.request.UpdateRequest
|
||||
import org.elasticsearch.action.ActionListener
|
||||
import org.elasticsearch.action.bulk.BackoffPolicy
|
||||
import org.elasticsearch.action.bulk.BulkItemResponse
|
||||
import org.elasticsearch.action.bulk.BulkProcessor
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
@@ -17,11 +20,13 @@ import org.elasticsearch.action.search.ClearScrollRequest
|
||||
import org.elasticsearch.action.search.SearchRequest
|
||||
import org.elasticsearch.action.search.SearchResponse
|
||||
import org.elasticsearch.action.search.SearchScrollRequest
|
||||
import org.elasticsearch.action.support.WriteRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.client.RestClient
|
||||
import org.elasticsearch.client.RestClientBuilder
|
||||
import org.elasticsearch.client.RestHighLevelClient
|
||||
import org.elasticsearch.common.unit.TimeValue
|
||||
import org.elasticsearch.common.xcontent.XContentType
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.search.Scroll
|
||||
import org.elasticsearch.search.SearchHit
|
||||
@@ -160,8 +165,17 @@ class ElasticUtils89 {
|
||||
}
|
||||
|
||||
@Override
|
||||
void afterBulk(long executionId, BulkRequest request,
|
||||
BulkResponse response) {
|
||||
public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
|
||||
if (response.hasFailures()) {
|
||||
for (BulkItemResponse itemResponse : response.getItems()) {
|
||||
if (itemResponse.isFailed()) {
|
||||
// 处理失败项
|
||||
String failureMessage = itemResponse.getFailureMessage();
|
||||
println failureMessage
|
||||
// 日志记录、告警或重试
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -205,6 +219,7 @@ class ElasticUtils89 {
|
||||
}
|
||||
BulkProcessor bulkProcessor = createBulkProcessor(host, client)
|
||||
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L))
|
||||
|
||||
SearchRequest searchRequest = new SearchRequest(indexName);
|
||||
searchRequest.scroll(scroll);
|
||||
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
|
||||
@@ -221,7 +236,7 @@ class ElasticUtils89 {
|
||||
// executorService.submit(new Runnable() {
|
||||
// @Override
|
||||
// void run() {
|
||||
try{
|
||||
// try{
|
||||
|
||||
if(closure.getMaximumNumberOfParameters() == 1){
|
||||
def update = closure.call(hit.getSourceAsMap())
|
||||
@@ -232,14 +247,16 @@ class ElasticUtils89 {
|
||||
def update = closure.call(hit.getId(), hit.getSourceAsMap())
|
||||
if(update){
|
||||
bulkProcessor.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap()))
|
||||
|
||||
}
|
||||
}
|
||||
}catch (Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
// }catch (Exception e){
|
||||
// e.printStackTrace()
|
||||
// }
|
||||
// }
|
||||
// })
|
||||
}
|
||||
|
||||
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
|
||||
scrollRequest.scroll(scroll);
|
||||
searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
|
||||
@@ -256,6 +273,120 @@ class ElasticUtils89 {
|
||||
client.close()
|
||||
}
|
||||
|
||||
static queryUpdate(String host, String indexName, QueryBuilder qb, Closure closure){
|
||||
RestHighLevelClient client = null
|
||||
if(host == "192.168.0.89"){
|
||||
client = getClient(host);
|
||||
}else if(host == "192.168.0.52"){
|
||||
client = getClientOther(host);
|
||||
}
|
||||
BulkProcessor bulkProcessor = createBulkProcessor(host, client)
|
||||
|
||||
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(30))
|
||||
// 2. 初始化滚动查询
|
||||
SearchRequest searchRequest = new SearchRequest(indexName);
|
||||
searchRequest.scroll(scroll);
|
||||
|
||||
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
|
||||
searchSourceBuilder.query(qb);
|
||||
searchRequest.source(searchSourceBuilder);
|
||||
|
||||
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
|
||||
String scrollId = searchResponse.getScrollId();
|
||||
SearchHit[] hits = searchResponse.getHits().getHits();
|
||||
|
||||
// 3. 批量操作容器
|
||||
BulkRequest bulkRequest = new BulkRequest();
|
||||
|
||||
// 4. 处理滚动结果
|
||||
while (hits.length > 0){
|
||||
for (SearchHit hit : hits) {
|
||||
// 示例修改逻辑:当满足条件时更新文档
|
||||
if(closure.getMaximumNumberOfParameters() == 2){
|
||||
def update = closure.call(hit.getId(), hit.getSourceAsMap())
|
||||
if(update){
|
||||
bulkRequest.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap()))
|
||||
}
|
||||
}
|
||||
// 批量提交控制
|
||||
if (bulkRequest.numberOfActions() >= 1) {
|
||||
bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(bulkRequest, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
System.err.println("批量操作部分失败: " + bulkResponse.buildFailureMessage());
|
||||
// 这里可以添加重试逻辑
|
||||
}
|
||||
bulkRequest = new BulkRequest(); // 重置容器
|
||||
}
|
||||
}
|
||||
|
||||
// 5. 获取下一批滚动结果
|
||||
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId).scroll(TimeValue.timeValueMinutes(5));
|
||||
searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
|
||||
scrollId = searchResponse.getScrollId();
|
||||
hits = searchResponse.getHits().getHits();
|
||||
}
|
||||
|
||||
// 6. 提交剩余批量操作
|
||||
if (bulkRequest.numberOfActions() > 0) {
|
||||
BulkResponse bulkResponse = client.bulk(bulkRequest, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
System.err.println("批量操作部分失败: " + bulkResponse.buildFailureMessage());
|
||||
}
|
||||
}
|
||||
|
||||
// 7. 清理滚动上下文
|
||||
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
|
||||
clearScrollRequest.addScrollId(scrollId);
|
||||
client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
|
||||
|
||||
bulkProcessor.close()
|
||||
bulkProcessor.awaitClose(1, TimeUnit.DAYS)
|
||||
client.close()
|
||||
}
|
||||
|
||||
|
||||
static updateEs(addEsDatasMap, host, indexName){
|
||||
BulkRequest request = new BulkRequest();
|
||||
int processedCount = 0;
|
||||
def client = null
|
||||
if(host && host == "192.168.0.89"){
|
||||
client = ElasticUtils89.getClient(host)
|
||||
}else if(host && host == "192.168.0.52"){
|
||||
client = ElasticUtils.getClient(host)
|
||||
}
|
||||
|
||||
addEsDatasMap.each{id, map->
|
||||
request.add(new IndexRequest(indexName, "_doc", id).source(map, XContentType.JSON));
|
||||
if (request.numberOfActions() >= 100) {
|
||||
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
// 处理错误
|
||||
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||
} else {
|
||||
processedCount += request.numberOfActions();
|
||||
println "Processed " + processedCount + " documents "+host
|
||||
}
|
||||
request = new BulkRequest();
|
||||
}
|
||||
}
|
||||
|
||||
if (request.numberOfActions() > 0) {
|
||||
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
// 处理错误
|
||||
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||
} else {
|
||||
processedCount += request.numberOfActions();
|
||||
println "Processed " + processedCount + " documents "+host
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
static def executeSimpleQuery(String host, String indexName, QueryBuilder qb, includes){
|
||||
|
||||
|
||||
Reference in New Issue
Block a user