+48
-12
@@ -3,8 +3,10 @@ package com.fangjia.script.dealPrice
|
||||
import com.fangjia.core.util.MongoUtils
|
||||
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.mongodb.AggregationOutput
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
import org.elasticsearch.action.index.IndexRequest
|
||||
@@ -21,23 +23,57 @@ class SynchDealPriceToEs {
|
||||
}
|
||||
|
||||
def run(){
|
||||
def batchsize = 1000
|
||||
def dbcursor = dealPriceConn.find(new BasicDBObject()).batchSize(batchsize)
|
||||
def ret = []
|
||||
def index = 0
|
||||
for(def d : dbcursor){
|
||||
ret << d
|
||||
if(ret.size()>=2000){
|
||||
save(ret)
|
||||
ret.clear()
|
||||
index ++
|
||||
println index
|
||||
def batchsize = 2000
|
||||
def citys = getCityCount()
|
||||
citys.each{city,count->
|
||||
def dbcursor = dealPriceConn.find(new BasicDBObject("city", city)).batchSize(batchsize)
|
||||
def ret = []
|
||||
for(def o : dbcursor){
|
||||
ret << o
|
||||
}
|
||||
save(ret)
|
||||
// def index = 0
|
||||
// for(def d : dbcursor){
|
||||
// ret << d
|
||||
// if(ret.size()>=2000){
|
||||
// save(ret)
|
||||
// ret.clear()
|
||||
// index ++
|
||||
// println index
|
||||
// }
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def getCityCount(){
|
||||
|
||||
BasicDBObject match = null;
|
||||
// 定义聚合管道,按 city 字段分组,并计算每个城市的数量
|
||||
DBObject groupFields = new BasicDBObject();
|
||||
|
||||
// match = new BasicDBObject('$match', new BasicDBObject("level", "城市"));
|
||||
groupFields.put("_id", new BasicDBObject("city",'$city'))
|
||||
|
||||
groupFields.put("count", new BasicDBObject('$sum', 1));
|
||||
DBObject group = new BasicDBObject('$group', groupFields);
|
||||
|
||||
// 执行聚合查询
|
||||
AggregationOutput output = dealPriceConn.aggregate(Arrays.asList(group));
|
||||
def map = [:]
|
||||
// 遍历结果
|
||||
for (DBObject result : output.results()) {
|
||||
def obj = (DBObject ) result.get("_id");
|
||||
if(obj.get("city")) {
|
||||
int count = (Integer) result.get("count");
|
||||
map.put(obj.get("city"), count)
|
||||
}
|
||||
}
|
||||
|
||||
return map;
|
||||
}
|
||||
|
||||
|
||||
|
||||
def save(rows){
|
||||
BulkRequest request = new BulkRequest();
|
||||
int batchSize = 500;
|
||||
|
||||
Reference in New Issue
Block a user