+83
@@ -0,0 +1,83 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
class SynchAdministrativeMappingDataBatch extends BaseBatch{
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection ADMINISTRATIVE_MAPPING_COLL_100 = MongoUtils.getCollectionByName2("fangjia", "administrative_mapping", MONGO100_URL)
|
||||
|
||||
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
runBatch(new SynchAdministrativeMappingDataBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def retMap = synchAdminiMappingTo100()
|
||||
synchToEs(retMap, "192.168.0.89")
|
||||
// synchToEs(retMap, "192.168.0.52")
|
||||
}
|
||||
|
||||
def synchAdminiMappingTo100(){
|
||||
//删除旧的,再同步新的
|
||||
ADMINISTRATIVE_MAPPING_COLL_100.remove(new BasicDBObject())
|
||||
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(new BasicDBObject()).batchSize(1000)
|
||||
def datas = []
|
||||
def retMap = [:]
|
||||
for(def d : dbcursor){
|
||||
datas << d
|
||||
retMap.put(d._id.toString(), d)
|
||||
if (datas.size() >= 2000) {
|
||||
ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
|
||||
println datas.size()
|
||||
datas.clear();
|
||||
}
|
||||
}
|
||||
|
||||
if (datas.size() >= 0) {
|
||||
ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
|
||||
datas.clear();
|
||||
}
|
||||
|
||||
return retMap;
|
||||
}
|
||||
|
||||
def synchToEs(devMap, host){
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery()
|
||||
|
||||
ElasticUtils89.executeQuery(host, "administrative_mapping", queryBuilder, { String id, doc->
|
||||
def mongoObj = devMap.get(id)
|
||||
def updateFieldCount = 0
|
||||
mongoObj.each{k,v->
|
||||
if(!["id","_id"].contains(k)){
|
||||
def val = doc.get(k)
|
||||
if(val!=v){
|
||||
println doc.name +" 修改 "+k+" "+ v+" "+id
|
||||
// updateValMap.put(k,val)
|
||||
updateFieldCount++
|
||||
doc.put(k,v)
|
||||
}
|
||||
}
|
||||
}
|
||||
if(mongoObj && updateFieldCount){
|
||||
//进行修改
|
||||
return true
|
||||
}else{
|
||||
if(!mongoObj){
|
||||
println "删除:"+id +" "+doc.name
|
||||
ElasticUtils.delete(id, host, "administrative_mapping")
|
||||
return false
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+1
-2
@@ -28,8 +28,7 @@ class SynchDevDataBatch extends BaseBatch{
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def devMap = synchDevTo100()
|
||||
// synchDevToEs(devMap)
|
||||
// synchDevEs(devMap, "192.168.0.89")
|
||||
synchDevEs(devMap, "192.168.0.89")
|
||||
synchDevEs(devMap, "192.168.0.52")
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user