公共方法修改

This commit is contained in:
2025-04-09 17:57:28 +08:00
parent 31cd461b8a
commit 1a09839cd3
@@ -1,5 +1,6 @@
package com.fangjia.script.gj.xiamen
import com.fangjia.script.util.ElasticUtils
import org.apache.http.HttpHost
import org.apache.http.auth.AuthScope
import org.apache.http.auth.UsernamePasswordCredentials
@@ -7,8 +8,10 @@ import org.apache.http.client.CredentialsProvider
import org.apache.http.client.config.RequestConfig
import org.apache.http.impl.client.BasicCredentialsProvider
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder
import org.apache.solr.client.solrj.request.UpdateRequest
import org.elasticsearch.action.ActionListener
import org.elasticsearch.action.bulk.BackoffPolicy
import org.elasticsearch.action.bulk.BulkItemResponse
import org.elasticsearch.action.bulk.BulkProcessor
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
@@ -17,11 +20,13 @@ import org.elasticsearch.action.search.ClearScrollRequest
import org.elasticsearch.action.search.SearchRequest
import org.elasticsearch.action.search.SearchResponse
import org.elasticsearch.action.search.SearchScrollRequest
import org.elasticsearch.action.support.WriteRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.client.RestClient
import org.elasticsearch.client.RestClientBuilder
import org.elasticsearch.client.RestHighLevelClient
import org.elasticsearch.common.unit.TimeValue
import org.elasticsearch.common.xcontent.XContentType
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.search.Scroll
import org.elasticsearch.search.SearchHit
@@ -160,8 +165,17 @@ class ElasticUtils89 {
}
@Override
void afterBulk(long executionId, BulkRequest request,
BulkResponse response) {
public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
if (response.hasFailures()) {
for (BulkItemResponse itemResponse : response.getItems()) {
if (itemResponse.isFailed()) {
// 处理失败项
String failureMessage = itemResponse.getFailureMessage();
println failureMessage
// 日志记录、告警或重试
}
}
}
}
@Override
@@ -205,6 +219,7 @@ class ElasticUtils89 {
}
BulkProcessor bulkProcessor = createBulkProcessor(host, client)
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L))
SearchRequest searchRequest = new SearchRequest(indexName);
searchRequest.scroll(scroll);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
@@ -221,7 +236,7 @@ class ElasticUtils89 {
// executorService.submit(new Runnable() {
// @Override
// void run() {
try{
// try{
if(closure.getMaximumNumberOfParameters() == 1){
def update = closure.call(hit.getSourceAsMap())
@@ -232,14 +247,16 @@ class ElasticUtils89 {
def update = closure.call(hit.getId(), hit.getSourceAsMap())
if(update){
bulkProcessor.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap()))
}
}
}catch (Exception e){
e.printStackTrace()
}
// }catch (Exception e){
// e.printStackTrace()
// }
// }
// })
}
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
scrollRequest.scroll(scroll);
searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
@@ -256,6 +273,120 @@ class ElasticUtils89 {
client.close()
}
static queryUpdate(String host, String indexName, QueryBuilder qb, Closure closure){
RestHighLevelClient client = null
if(host == "192.168.0.89"){
client = getClient(host);
}else if(host == "192.168.0.52"){
client = getClientOther(host);
}
BulkProcessor bulkProcessor = createBulkProcessor(host, client)
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(30))
// 2. 初始化滚动查询
SearchRequest searchRequest = new SearchRequest(indexName);
searchRequest.scroll(scroll);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(qb);
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
String scrollId = searchResponse.getScrollId();
SearchHit[] hits = searchResponse.getHits().getHits();
// 3. 批量操作容器
BulkRequest bulkRequest = new BulkRequest();
// 4. 处理滚动结果
while (hits.length > 0){
for (SearchHit hit : hits) {
// 示例修改逻辑:当满足条件时更新文档
if(closure.getMaximumNumberOfParameters() == 2){
def update = closure.call(hit.getId(), hit.getSourceAsMap())
if(update){
bulkRequest.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap()))
}
}
// 批量提交控制
if (bulkRequest.numberOfActions() >= 1) {
bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(bulkRequest, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
System.err.println("批量操作部分失败: " + bulkResponse.buildFailureMessage());
// 这里可以添加重试逻辑
}
bulkRequest = new BulkRequest(); // 重置容器
}
}
// 5. 获取下一批滚动结果
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId).scroll(TimeValue.timeValueMinutes(5));
searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
scrollId = searchResponse.getScrollId();
hits = searchResponse.getHits().getHits();
}
// 6. 提交剩余批量操作
if (bulkRequest.numberOfActions() > 0) {
BulkResponse bulkResponse = client.bulk(bulkRequest, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
System.err.println("批量操作部分失败: " + bulkResponse.buildFailureMessage());
}
}
// 7. 清理滚动上下文
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
clearScrollRequest.addScrollId(scrollId);
client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
bulkProcessor.close()
bulkProcessor.awaitClose(1, TimeUnit.DAYS)
client.close()
}
static updateEs(addEsDatasMap, host, indexName){
BulkRequest request = new BulkRequest();
int processedCount = 0;
def client = null
if(host && host == "192.168.0.89"){
client = ElasticUtils89.getClient(host)
}else if(host && host == "192.168.0.52"){
client = ElasticUtils.getClient(host)
}
addEsDatasMap.each{id, map->
request.add(new IndexRequest(indexName, "_doc", id).source(map, XContentType.JSON));
if (request.numberOfActions() >= 100) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents "+host
}
request = new BulkRequest();
}
}
if (request.numberOfActions() > 0) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents "+host
}
}
}
static def executeSimpleQuery(String host, String indexName, QueryBuilder qb, includes){