Merge branch 'script_gj' into 'master'

update

See merge request !1063
This commit is contained in:
2025-01-06 13:31:55 +08:00
@@ -3,8 +3,10 @@ package com.fangjia.script.dealPrice
import com.fangjia.core.util.MongoUtils
import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.util.ElasticUtils
import com.mongodb.AggregationOutput
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.action.index.IndexRequest
@@ -21,23 +23,57 @@ class SynchDealPriceToEs {
}
def run(){
def batchsize = 1000
def dbcursor = dealPriceConn.find(new BasicDBObject()).batchSize(batchsize)
def ret = []
def index = 0
for(def d : dbcursor){
ret << d
if(ret.size()>=2000){
save(ret)
ret.clear()
index ++
println index
def batchsize = 2000
def citys = getCityCount()
citys.each{city,count->
def dbcursor = dealPriceConn.find(new BasicDBObject("city", city)).batchSize(batchsize)
def ret = []
for(def o : dbcursor){
ret << o
}
save(ret)
// def index = 0
// for(def d : dbcursor){
// ret << d
// if(ret.size()>=2000){
// save(ret)
// ret.clear()
// index ++
// println index
// }
// }
}
}
def getCityCount(){
BasicDBObject match = null;
// 定义聚合管道,按 city 字段分组,并计算每个城市的数量
DBObject groupFields = new BasicDBObject();
// match = new BasicDBObject('$match', new BasicDBObject("level", "城市"));
groupFields.put("_id", new BasicDBObject("city",'$city'))
groupFields.put("count", new BasicDBObject('$sum', 1));
DBObject group = new BasicDBObject('$group', groupFields);
// 执行聚合查询
AggregationOutput output = dealPriceConn.aggregate(Arrays.asList(group));
def map = [:]
// 遍历结果
for (DBObject result : output.results()) {
def obj = (DBObject ) result.get("_id");
if(obj.get("city")) {
int count = (Integer) result.get("count");
map.put(obj.get("city"), count)
}
}
return map;
}
def save(rows){
BulkRequest request = new BulkRequest();
int batchSize = 500;