diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy index 3bc2f080a..aeac02b63 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy @@ -6,6 +6,10 @@ import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.xiamen.ElasticUtils89 import com.fangjia.script.util.DbUtils import com.mongodb.BasicDBObject +import org.elasticsearch.action.bulk.BulkRequest +import org.elasticsearch.action.update.UpdateRequest +import org.elasticsearch.client.RequestOptions +import org.elasticsearch.common.xcontent.XContentType import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilders @@ -162,7 +166,7 @@ class RegionStandardBatch extends BaseBatch{ QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)) def rets = [] - def datas = ElasticUtils89.executeSimpleQuery("192.168.0.89", "didata_map_poi_shards", queryBuilder) + def datas = ElasticUtils89.executeSimpleQuery("192.168.0.89", "didata_map_poi", queryBuilder) for(def d: datas){ def o = new BasicDBObject() o.put("city", d.city) @@ -170,39 +174,95 @@ class RegionStandardBatch extends BaseBatch{ o.put("name", d.name) o.put("township", d.township) o.put("location", d.location) + o.put("status", d.status) + o.put("id", d.id) rets << o - }) + } return rets } def poiHaveRegionProcess(city, set, rMap, datas){ + def results = [] for(def d : datas){ - if(set.contains(d.region)){ + if(set.contains(d.district)){ continue } def updateRegion = null for(def key : set){ def alias = rMap.get(key) - if(alias.contains(d.region)){ + if(alias.contains(d.district)){ updateRegion = key break; } } if(updateRegion){ - def udata = new BasicDBObject() - udata.put("region", updateRegion) - udata.put("m_date", new Date()) - println d.city +" "+d.region+" "+d.name +" 区域修改为:"+updateRegion + def m = [:] + m.put(d.id, ["district":updateRegion]) + results << m + if(results.size()%1==0){ + updateRegionByEs(results) + results.clear() + } + println d.city +" "+d.district+" "+d.name +" "+d.status+" 区域修改为:"+updateRegion }else{ - def udata = new BasicDBObject() - udata.put("status", 1) - udata.put("m_date", new Date()) - println d.city +" "+d.region+" "+d.name +" 状态修改为:删除" + def m = [:] + m.put(d.id, ["status":1]) + results << m + if(results.size()%1==0){ + updateRegionByEs(results) + results.clear() + } + + println d.city +" "+d.district+" "+d.name +" "+d.status+" 状态修改为:删除" } } + + if(results.size()%1>0){ + updateRegionByEs(results) + results.clear() + } + } + + + + def updateRegionByEs(rows){ + def client = ElasticUtils89.getClient("192.168.0.89") + def bulkRequest = new BulkRequest() + + rows.each { row -> + def id = row.keySet().iterator().next() + def doc = row.get(id) + + def updateRequest = new UpdateRequest("didata_map_poi", id) + updateRequest.doc(doc, XContentType.JSON) + bulkRequest.add(updateRequest) + } + + try { + // 执行批量更新请求 + def bulkResponse = client.bulk(bulkRequest, RequestOptions.DEFAULT) + + // 检查响应 + if (bulkResponse.hasFailures()) { + println("批量更新中有失败的操作:") + bulkResponse.buildFailureMessage().each { println(it) } + } else { + println("批量更新成功!") + } + } catch (Exception e) { + e.printStackTrace() + } finally { + // 关闭客户端 + try { + client.close() + } catch (Exception e) { + e.printStackTrace() + } + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy index db5362ca7..badba650a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy @@ -236,13 +236,13 @@ class ElasticUtils89 { searchRequest.scroll(scroll); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.query(qb); - searchSourceBuilder.size(100) +// searchSourceBuilder.size(1000) searchRequest.source(searchSourceBuilder); SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); String scrollId = searchResponse.getScrollId(); SearchHit[] searchHits = searchResponse.getHits().getHits(); def results = [] - while (searchHits != null && searchHits.length > 0) { +// while (searchHits != null && searchHits.length > 0) { for (int k=0; k < searchHits.size(); k++) { SearchHit searchHit = searchHits[k] searchHit.getSourceAsMap().put("id", searchHit.getId()); @@ -256,7 +256,7 @@ class ElasticUtils89 { // searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT); // scrollId = searchResponse.getScrollId(); // searchHits = searchResponse.getHits().getHits(); - } +// } ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); clearScrollRequest.addScrollId(scrollId); client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);