Merge branch 'script_gj' into 'master'

update

See merge request !1063
This commit is contained in:
2025-01-06 13:31:55 +08:00
@@ -3,8 +3,10 @@ package com.fangjia.script.dealPrice
import com.fangjia.core.util.MongoUtils import com.fangjia.core.util.MongoUtils
import com.fangjia.script.gj.xiamen.ElasticUtils89 import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ElasticUtils
import com.mongodb.AggregationOutput
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.DBCollection import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.elasticsearch.action.bulk.BulkRequest import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.action.index.IndexRequest import org.elasticsearch.action.index.IndexRequest
@@ -21,23 +23,57 @@ class SynchDealPriceToEs {
} }
def run(){ def run(){
def batchsize = 1000 def batchsize = 2000
def dbcursor = dealPriceConn.find(new BasicDBObject()).batchSize(batchsize) def citys = getCityCount()
def ret = [] citys.each{city,count->
def index = 0 def dbcursor = dealPriceConn.find(new BasicDBObject("city", city)).batchSize(batchsize)
for(def d : dbcursor){ def ret = []
ret << d for(def o : dbcursor){
if(ret.size()>=2000){ ret << o
save(ret) }
ret.clear() save(ret)
index ++ // def index = 0
println index // for(def d : dbcursor){
// ret << d
// if(ret.size()>=2000){
// save(ret)
// ret.clear()
// index ++
// println index
// }
// }
}
}
def getCityCount(){
BasicDBObject match = null;
// 定义聚合管道,按 city 字段分组,并计算每个城市的数量
DBObject groupFields = new BasicDBObject();
// match = new BasicDBObject('$match', new BasicDBObject("level", "城市"));
groupFields.put("_id", new BasicDBObject("city",'$city'))
groupFields.put("count", new BasicDBObject('$sum', 1));
DBObject group = new BasicDBObject('$group', groupFields);
// 执行聚合查询
AggregationOutput output = dealPriceConn.aggregate(Arrays.asList(group));
def map = [:]
// 遍历结果
for (DBObject result : output.results()) {
def obj = (DBObject ) result.get("_id");
if(obj.get("city")) {
int count = (Integer) result.get("count");
map.put(obj.get("city"), count)
} }
} }
return map;
} }
def save(rows){ def save(rows){
BulkRequest request = new BulkRequest(); BulkRequest request = new BulkRequest();
int batchSize = 500; int batchSize = 500;