From 1a09839cd310d7c4bb14b501a10c6e4d7e6d7755 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Wed, 9 Apr 2025 17:57:28 +0800 Subject: [PATCH] =?UTF-8?q?=E5=85=AC=E5=85=B1=E6=96=B9=E6=B3=95=E4=BF=AE?= =?UTF-8?q?=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/gj/xiamen/ElasticUtils89.groovy | 143 +++++++++++++++++- 1 file changed, 137 insertions(+), 6 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy index b1b772b92..dd2c2e5e1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.gj.xiamen +import com.fangjia.script.util.ElasticUtils import org.apache.http.HttpHost import org.apache.http.auth.AuthScope import org.apache.http.auth.UsernamePasswordCredentials @@ -7,8 +8,10 @@ import org.apache.http.client.CredentialsProvider import org.apache.http.client.config.RequestConfig import org.apache.http.impl.client.BasicCredentialsProvider import org.apache.http.impl.nio.client.HttpAsyncClientBuilder +import org.apache.solr.client.solrj.request.UpdateRequest import org.elasticsearch.action.ActionListener import org.elasticsearch.action.bulk.BackoffPolicy +import org.elasticsearch.action.bulk.BulkItemResponse import org.elasticsearch.action.bulk.BulkProcessor import org.elasticsearch.action.bulk.BulkRequest import org.elasticsearch.action.bulk.BulkResponse @@ -17,11 +20,13 @@ import org.elasticsearch.action.search.ClearScrollRequest import org.elasticsearch.action.search.SearchRequest import org.elasticsearch.action.search.SearchResponse import org.elasticsearch.action.search.SearchScrollRequest +import org.elasticsearch.action.support.WriteRequest import org.elasticsearch.client.RequestOptions import org.elasticsearch.client.RestClient import org.elasticsearch.client.RestClientBuilder import org.elasticsearch.client.RestHighLevelClient import org.elasticsearch.common.unit.TimeValue +import org.elasticsearch.common.xcontent.XContentType import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.search.Scroll import org.elasticsearch.search.SearchHit @@ -160,8 +165,17 @@ class ElasticUtils89 { } @Override - void afterBulk(long executionId, BulkRequest request, - BulkResponse response) { + public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { + if (response.hasFailures()) { + for (BulkItemResponse itemResponse : response.getItems()) { + if (itemResponse.isFailed()) { + // 处理失败项 + String failureMessage = itemResponse.getFailureMessage(); + println failureMessage + // 日志记录、告警或重试 + } + } + } } @Override @@ -205,6 +219,7 @@ class ElasticUtils89 { } BulkProcessor bulkProcessor = createBulkProcessor(host, client) final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L)) + SearchRequest searchRequest = new SearchRequest(indexName); searchRequest.scroll(scroll); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); @@ -221,7 +236,7 @@ class ElasticUtils89 { // executorService.submit(new Runnable() { // @Override // void run() { - try{ +// try{ if(closure.getMaximumNumberOfParameters() == 1){ def update = closure.call(hit.getSourceAsMap()) @@ -232,14 +247,16 @@ class ElasticUtils89 { def update = closure.call(hit.getId(), hit.getSourceAsMap()) if(update){ bulkProcessor.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap())) + } } - }catch (Exception e){ - e.printStackTrace() - } +// }catch (Exception e){ +// e.printStackTrace() +// } // } // }) } + SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); scrollRequest.scroll(scroll); searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT); @@ -256,6 +273,120 @@ class ElasticUtils89 { client.close() } + static queryUpdate(String host, String indexName, QueryBuilder qb, Closure closure){ + RestHighLevelClient client = null + if(host == "192.168.0.89"){ + client = getClient(host); + }else if(host == "192.168.0.52"){ + client = getClientOther(host); + } + BulkProcessor bulkProcessor = createBulkProcessor(host, client) + + final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(30)) + // 2. 初始化滚动查询 + SearchRequest searchRequest = new SearchRequest(indexName); + searchRequest.scroll(scroll); + + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(qb); + searchRequest.source(searchSourceBuilder); + + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + String scrollId = searchResponse.getScrollId(); + SearchHit[] hits = searchResponse.getHits().getHits(); + + // 3. 批量操作容器 + BulkRequest bulkRequest = new BulkRequest(); + + // 4. 处理滚动结果 + while (hits.length > 0){ + for (SearchHit hit : hits) { + // 示例修改逻辑:当满足条件时更新文档 + if(closure.getMaximumNumberOfParameters() == 2){ + def update = closure.call(hit.getId(), hit.getSourceAsMap()) + if(update){ + bulkRequest.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap())) + } + } + // 批量提交控制 + if (bulkRequest.numberOfActions() >= 1) { + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = client.bulk(bulkRequest, RequestOptions.DEFAULT); + if (bulkResponse.hasFailures()) { + System.err.println("批量操作部分失败: " + bulkResponse.buildFailureMessage()); + // 这里可以添加重试逻辑 + } + bulkRequest = new BulkRequest(); // 重置容器 + } + } + + // 5. 获取下一批滚动结果 + SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId).scroll(TimeValue.timeValueMinutes(5)); + searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT); + scrollId = searchResponse.getScrollId(); + hits = searchResponse.getHits().getHits(); + } + + // 6. 提交剩余批量操作 + if (bulkRequest.numberOfActions() > 0) { + BulkResponse bulkResponse = client.bulk(bulkRequest, RequestOptions.DEFAULT); + if (bulkResponse.hasFailures()) { + System.err.println("批量操作部分失败: " + bulkResponse.buildFailureMessage()); + } + } + + // 7. 清理滚动上下文 + ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); + clearScrollRequest.addScrollId(scrollId); + client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT); + + bulkProcessor.close() + bulkProcessor.awaitClose(1, TimeUnit.DAYS) + client.close() + } + + + static updateEs(addEsDatasMap, host, indexName){ + BulkRequest request = new BulkRequest(); + int processedCount = 0; + def client = null + if(host && host == "192.168.0.89"){ + client = ElasticUtils89.getClient(host) + }else if(host && host == "192.168.0.52"){ + client = ElasticUtils.getClient(host) + } + + addEsDatasMap.each{id, map-> + request.add(new IndexRequest(indexName, "_doc", id).source(map, XContentType.JSON)); + if (request.numberOfActions() >= 100) { + request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); + if (bulkResponse.hasFailures()) { + // 处理错误 + println "Bulk request has failures: " + bulkResponse.buildFailureMessage() + } else { + processedCount += request.numberOfActions(); + println "Processed " + processedCount + " documents "+host + } + request = new BulkRequest(); + } + } + + if (request.numberOfActions() > 0) { + request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); + if (bulkResponse.hasFailures()) { + // 处理错误 + println "Bulk request has failures: " + bulkResponse.buildFailureMessage() + } else { + processedCount += request.numberOfActions(); + println "Processed " + processedCount + " documents "+host + } + } + } + + + static def executeSimpleQuery(String host, String indexName, QueryBuilder qb, includes){