优化es获取数据量

This commit is contained in:
Jian Shen
2022-07-02 15:49:24 +08:00
parent acc809b3ae
commit 2fec35aa97
2 changed files with 2 additions and 3 deletions
@@ -86,7 +86,6 @@ class GetPoiFromCommunity {
static def findCommunities(){
QueryBuilder queryBuilder = QueryBuilders.boolQuery()
.filter(QueryBuilders.termQuery("city", "东莞"))
.mustNot(QueryBuilders.termQuery("crawl_flag", 0))
List<Community> communities = EsUtils.scrollQuery("community", queryBuilder, Community.class)
return communities
@@ -35,7 +35,7 @@ class EsUtils {
* @param <T>
* @return
*/
public static <T extends MyId> List<T> scrollQuery(String index, QueryBuilder boolQuery, Class<T> clazz) {
public static <T extends MyId> List<T> scrollQuery(String index, QueryBuilder boolQuery, Class<T> clazz, def limitSize=300000) {
List<T> result = new ArrayList<>();
// 存活时间,当索引数据量特别大时,出现超时可能性大,此值适当调大
Scroll scroll = new Scroll(TimeValue.timeValueMinutes(30L));
@@ -61,7 +61,7 @@ class EsUtils {
}
}
// 遍历搜索命中的数据,直到没有数据
while (searchHits != null && searchHits.length > 0) {
while (searchHits != null && searchHits.length > 0 && result.size() < limitSize) {
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
scrollRequest.scroll(scroll);
try {