+48
-12
@@ -3,8 +3,10 @@ package com.fangjia.script.dealPrice
|
|||||||
import com.fangjia.core.util.MongoUtils
|
import com.fangjia.core.util.MongoUtils
|
||||||
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
||||||
import com.fangjia.script.util.ElasticUtils
|
import com.fangjia.script.util.ElasticUtils
|
||||||
|
import com.mongodb.AggregationOutput
|
||||||
import com.mongodb.BasicDBObject
|
import com.mongodb.BasicDBObject
|
||||||
import com.mongodb.DBCollection
|
import com.mongodb.DBCollection
|
||||||
|
import com.mongodb.DBObject
|
||||||
import org.elasticsearch.action.bulk.BulkRequest
|
import org.elasticsearch.action.bulk.BulkRequest
|
||||||
import org.elasticsearch.action.bulk.BulkResponse
|
import org.elasticsearch.action.bulk.BulkResponse
|
||||||
import org.elasticsearch.action.index.IndexRequest
|
import org.elasticsearch.action.index.IndexRequest
|
||||||
@@ -21,23 +23,57 @@ class SynchDealPriceToEs {
|
|||||||
}
|
}
|
||||||
|
|
||||||
def run(){
|
def run(){
|
||||||
def batchsize = 1000
|
def batchsize = 2000
|
||||||
def dbcursor = dealPriceConn.find(new BasicDBObject()).batchSize(batchsize)
|
def citys = getCityCount()
|
||||||
def ret = []
|
citys.each{city,count->
|
||||||
def index = 0
|
def dbcursor = dealPriceConn.find(new BasicDBObject("city", city)).batchSize(batchsize)
|
||||||
for(def d : dbcursor){
|
def ret = []
|
||||||
ret << d
|
for(def o : dbcursor){
|
||||||
if(ret.size()>=2000){
|
ret << o
|
||||||
save(ret)
|
}
|
||||||
ret.clear()
|
save(ret)
|
||||||
index ++
|
// def index = 0
|
||||||
println index
|
// for(def d : dbcursor){
|
||||||
|
// ret << d
|
||||||
|
// if(ret.size()>=2000){
|
||||||
|
// save(ret)
|
||||||
|
// ret.clear()
|
||||||
|
// index ++
|
||||||
|
// println index
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def getCityCount(){
|
||||||
|
|
||||||
|
BasicDBObject match = null;
|
||||||
|
// 定义聚合管道,按 city 字段分组,并计算每个城市的数量
|
||||||
|
DBObject groupFields = new BasicDBObject();
|
||||||
|
|
||||||
|
// match = new BasicDBObject('$match', new BasicDBObject("level", "城市"));
|
||||||
|
groupFields.put("_id", new BasicDBObject("city",'$city'))
|
||||||
|
|
||||||
|
groupFields.put("count", new BasicDBObject('$sum', 1));
|
||||||
|
DBObject group = new BasicDBObject('$group', groupFields);
|
||||||
|
|
||||||
|
// 执行聚合查询
|
||||||
|
AggregationOutput output = dealPriceConn.aggregate(Arrays.asList(group));
|
||||||
|
def map = [:]
|
||||||
|
// 遍历结果
|
||||||
|
for (DBObject result : output.results()) {
|
||||||
|
def obj = (DBObject ) result.get("_id");
|
||||||
|
if(obj.get("city")) {
|
||||||
|
int count = (Integer) result.get("count");
|
||||||
|
map.put(obj.get("city"), count)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
return map;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def save(rows){
|
def save(rows){
|
||||||
BulkRequest request = new BulkRequest();
|
BulkRequest request = new BulkRequest();
|
||||||
int batchSize = 500;
|
int batchSize = 500;
|
||||||
|
|||||||
Reference in New Issue
Block a user