diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy index aeac02b63..8533557df 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy @@ -38,7 +38,7 @@ class RegionStandardBatch extends BaseBatch{ seaweedRegionStandardByCat("shop") } def poiRegionStandard(){ - def citys = ["三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德"]//CommonDatautils.findAllCity() + def citys = ["三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德","重庆"]//CommonDatautils.findAllCity() for(def city : citys){ def data = fullCityMap.get(city) @@ -200,7 +200,7 @@ class RegionStandardBatch extends BaseBatch{ def m = [:] m.put(d.id, ["district":updateRegion]) results << m - if(results.size()%1==0){ + if(results.size()%100==0){ updateRegionByEs(results) results.clear() } @@ -210,7 +210,7 @@ class RegionStandardBatch extends BaseBatch{ def m = [:] m.put(d.id, ["status":1]) results << m - if(results.size()%1==0){ + if(results.size()%100==0){ updateRegionByEs(results) results.clear() } @@ -220,10 +220,11 @@ class RegionStandardBatch extends BaseBatch{ } } - if(results.size()%1>0){ + if(results.size()>0){ updateRegionByEs(results) results.clear() } + println city +" 处理完成" } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy index badba650a..e31ded3ec 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy @@ -242,21 +242,21 @@ class ElasticUtils89 { String scrollId = searchResponse.getScrollId(); SearchHit[] searchHits = searchResponse.getHits().getHits(); def results = [] -// while (searchHits != null && searchHits.length > 0) { + while (searchHits != null && searchHits.length > 0) { for (int k=0; k < searchHits.size(); k++) { SearchHit searchHit = searchHits[k] searchHit.getSourceAsMap().put("id", searchHit.getId()); results << searchHit.getSourceAsMap() } -// if(results.size()%100000==0){ -// println "."+ new Date() -// } -// SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); -// scrollRequest.scroll(scroll); -// searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT); -// scrollId = searchResponse.getScrollId(); -// searchHits = searchResponse.getHits().getHits(); -// } + if(results.size()%100000==0){ + println "."+ new Date() + } + SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); + scrollRequest.scroll(scroll); + searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT); + scrollId = searchResponse.getScrollId(); + searchHits = searchResponse.getHits().getHits(); + } ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); clearScrollRequest.addScrollId(scrollId); client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);