+83
@@ -0,0 +1,83 @@
|
||||
package com.fangjia.script.dealPrice
|
||||
|
||||
import com.fangjia.core.util.MongoUtils
|
||||
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
import org.elasticsearch.action.index.IndexRequest
|
||||
import org.elasticsearch.action.support.WriteRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.common.xcontent.XContentType
|
||||
|
||||
class SynchDealPriceToEs {
|
||||
static DBCollection dealPriceConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43",28017)
|
||||
|
||||
static main(args) {
|
||||
def task = new SynchDealPriceToEs()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def batchsize = 1000
|
||||
def dbcursor = dealPriceConn.find(new BasicDBObject()).batchSize(batchsize)
|
||||
def ret = []
|
||||
def index = 0
|
||||
for(def d : dbcursor){
|
||||
ret << d
|
||||
if(ret.size()>=2000){
|
||||
save(ret)
|
||||
ret.clear()
|
||||
index ++
|
||||
println index
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
def save(rows){
|
||||
BulkRequest request = new BulkRequest();
|
||||
int batchSize = 500;
|
||||
int processedCount = 0;
|
||||
def client = ElasticUtils.getClient("192.168.0.206")
|
||||
|
||||
for(def map : rows){
|
||||
// def map = rows.get(key)
|
||||
def data_id = map._id.toString()
|
||||
map.remove("_id")
|
||||
request.add(new IndexRequest("deal_price", "_doc", data_id).source(map, XContentType.JSON));
|
||||
|
||||
// 批量提交数据
|
||||
if (request.numberOfActions() >= batchSize) {
|
||||
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
// 处理错误
|
||||
print "Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||
} else {
|
||||
processedCount += request.numberOfActions();
|
||||
println "Processed " + processedCount + " documents "+new Date().format("yyyy-MM-dd HH:mm:ss")
|
||||
}
|
||||
|
||||
request = new BulkRequest();
|
||||
}
|
||||
}
|
||||
|
||||
if (request.numberOfActions() > 0) {
|
||||
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
// 处理错误
|
||||
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||
} else {
|
||||
processedCount += request.numberOfActions();
|
||||
println "Processed " + processedCount + " documents"
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user