同步成交

This commit is contained in:
2025-01-06 07:56:47 +08:00
parent ca239765c9
commit 761f6221c1
@@ -0,0 +1,83 @@
package com.fangjia.script.dealPrice
import com.fangjia.core.util.MongoUtils
import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.util.ElasticUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.action.support.WriteRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.common.xcontent.XContentType
class SynchDealPriceToEs {
static DBCollection dealPriceConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43",28017)
static main(args) {
def task = new SynchDealPriceToEs()
task.run()
}
def run(){
def batchsize = 1000
def dbcursor = dealPriceConn.find(new BasicDBObject()).batchSize(batchsize)
def ret = []
def index = 0
for(def d : dbcursor){
ret << d
if(ret.size()>=2000){
save(ret)
ret.clear()
index ++
println index
}
}
}
def save(rows){
BulkRequest request = new BulkRequest();
int batchSize = 500;
int processedCount = 0;
def client = ElasticUtils.getClient("192.168.0.206")
for(def map : rows){
// def map = rows.get(key)
def data_id = map._id.toString()
map.remove("_id")
request.add(new IndexRequest("deal_price", "_doc", data_id).source(map, XContentType.JSON));
// 批量提交数据
if (request.numberOfActions() >= batchSize) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
print "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents "+new Date().format("yyyy-MM-dd HH:mm:ss")
}
request = new BulkRequest();
}
}
if (request.numberOfActions() > 0) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents"
}
}
}
}