优化es获取数据量
This commit is contained in:
-1
@@ -86,7 +86,6 @@ class GetPoiFromCommunity {
|
||||
|
||||
static def findCommunities(){
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery()
|
||||
.filter(QueryBuilders.termQuery("city", "东莞"))
|
||||
.mustNot(QueryBuilders.termQuery("crawl_flag", 0))
|
||||
List<Community> communities = EsUtils.scrollQuery("community", queryBuilder, Community.class)
|
||||
return communities
|
||||
|
||||
+2
-2
@@ -35,7 +35,7 @@ class EsUtils {
|
||||
* @param <T>
|
||||
* @return
|
||||
*/
|
||||
public static <T extends MyId> List<T> scrollQuery(String index, QueryBuilder boolQuery, Class<T> clazz) {
|
||||
public static <T extends MyId> List<T> scrollQuery(String index, QueryBuilder boolQuery, Class<T> clazz, def limitSize=300000) {
|
||||
List<T> result = new ArrayList<>();
|
||||
// 存活时间,当索引数据量特别大时,出现超时可能性大,此值适当调大
|
||||
Scroll scroll = new Scroll(TimeValue.timeValueMinutes(30L));
|
||||
@@ -61,7 +61,7 @@ class EsUtils {
|
||||
}
|
||||
}
|
||||
// 遍历搜索命中的数据,直到没有数据
|
||||
while (searchHits != null && searchHits.length > 0) {
|
||||
while (searchHits != null && searchHits.length > 0 && result.size() < limitSize) {
|
||||
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
|
||||
scrollRequest.scroll(scroll);
|
||||
try {
|
||||
|
||||
Reference in New Issue
Block a user