From 96e13d215ff1ef5d6a2fd891ebd37f272d88bd21 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Mon, 6 Jan 2025 13:29:30 +0800 Subject: [PATCH] update --- .../dealPrice/SynchDealPriceToEs.groovy | 60 +++++++++++++++---- 1 file changed, 48 insertions(+), 12 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/dealPrice/SynchDealPriceToEs.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/dealPrice/SynchDealPriceToEs.groovy index 7d06921dc..13ea5e3de 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/dealPrice/SynchDealPriceToEs.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/dealPrice/SynchDealPriceToEs.groovy @@ -3,8 +3,10 @@ package com.fangjia.script.dealPrice import com.fangjia.core.util.MongoUtils import com.fangjia.script.gj.xiamen.ElasticUtils89 import com.fangjia.script.util.ElasticUtils +import com.mongodb.AggregationOutput import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import com.mongodb.DBObject import org.elasticsearch.action.bulk.BulkRequest import org.elasticsearch.action.bulk.BulkResponse import org.elasticsearch.action.index.IndexRequest @@ -21,23 +23,57 @@ class SynchDealPriceToEs { } def run(){ - def batchsize = 1000 - def dbcursor = dealPriceConn.find(new BasicDBObject()).batchSize(batchsize) - def ret = [] - def index = 0 - for(def d : dbcursor){ - ret << d - if(ret.size()>=2000){ - save(ret) - ret.clear() - index ++ - println index + def batchsize = 2000 + def citys = getCityCount() + citys.each{city,count-> + def dbcursor = dealPriceConn.find(new BasicDBObject("city", city)).batchSize(batchsize) + def ret = [] + for(def o : dbcursor){ + ret << o + } + save(ret) +// def index = 0 +// for(def d : dbcursor){ +// ret << d +// if(ret.size()>=2000){ +// save(ret) +// ret.clear() +// index ++ +// println index +// } +// } + } + } + + + def getCityCount(){ + + BasicDBObject match = null; + // 定义聚合管道,按 city 字段分组,并计算每个城市的数量 + DBObject groupFields = new BasicDBObject(); + +// match = new BasicDBObject('$match', new BasicDBObject("level", "城市")); + groupFields.put("_id", new BasicDBObject("city",'$city')) + + groupFields.put("count", new BasicDBObject('$sum', 1)); + DBObject group = new BasicDBObject('$group', groupFields); + + // 执行聚合查询 + AggregationOutput output = dealPriceConn.aggregate(Arrays.asList(group)); + def map = [:] + // 遍历结果 + for (DBObject result : output.results()) { + def obj = (DBObject ) result.get("_id"); + if(obj.get("city")) { + int count = (Integer) result.get("count"); + map.put(obj.get("city"), count) } } - + return map; } + def save(rows){ BulkRequest request = new BulkRequest(); int batchSize = 500;