diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/dealPrice/SynchDealPriceToEs.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/dealPrice/SynchDealPriceToEs.groovy new file mode 100644 index 000000000..7d06921dc --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/dealPrice/SynchDealPriceToEs.groovy @@ -0,0 +1,83 @@ +package com.fangjia.script.dealPrice + +import com.fangjia.core.util.MongoUtils +import com.fangjia.script.gj.xiamen.ElasticUtils89 +import com.fangjia.script.util.ElasticUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.elasticsearch.action.bulk.BulkRequest +import org.elasticsearch.action.bulk.BulkResponse +import org.elasticsearch.action.index.IndexRequest +import org.elasticsearch.action.support.WriteRequest +import org.elasticsearch.client.RequestOptions +import org.elasticsearch.common.xcontent.XContentType + +class SynchDealPriceToEs { + static DBCollection dealPriceConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43",28017) + + static main(args) { + def task = new SynchDealPriceToEs() + task.run() + } + + def run(){ + def batchsize = 1000 + def dbcursor = dealPriceConn.find(new BasicDBObject()).batchSize(batchsize) + def ret = [] + def index = 0 + for(def d : dbcursor){ + ret << d + if(ret.size()>=2000){ + save(ret) + ret.clear() + index ++ + println index + } + } + + } + + + def save(rows){ + BulkRequest request = new BulkRequest(); + int batchSize = 500; + int processedCount = 0; + def client = ElasticUtils.getClient("192.168.0.206") + + for(def map : rows){ +// def map = rows.get(key) + def data_id = map._id.toString() + map.remove("_id") + request.add(new IndexRequest("deal_price", "_doc", data_id).source(map, XContentType.JSON)); + + // 批量提交数据 + if (request.numberOfActions() >= batchSize) { + request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); + if (bulkResponse.hasFailures()) { + // 处理错误 + print "Bulk request has failures: " + bulkResponse.buildFailureMessage() + } else { + processedCount += request.numberOfActions(); + println "Processed " + processedCount + " documents "+new Date().format("yyyy-MM-dd HH:mm:ss") + } + + request = new BulkRequest(); + } + } + + if (request.numberOfActions() > 0) { + request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); + if (bulkResponse.hasFailures()) { + // 处理错误 + println "Bulk request has failures: " + bulkResponse.buildFailureMessage() + } else { + processedCount += request.numberOfActions(); + println "Processed " + processedCount + " documents" + } + } + + } + +}