同步脚本修改

This commit is contained in:
2025-04-08 16:05:51 +08:00
parent f0548b3fe0
commit d73ee6e1ea
2 changed files with 84 additions and 2 deletions
@@ -0,0 +1,83 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.BaseBatch
import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
class SynchAdministrativeMappingDataBatch extends BaseBatch{
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection ADMINISTRATIVE_MAPPING_COLL_100 = MongoUtils.getCollectionByName2("fangjia", "administrative_mapping", MONGO100_URL)
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
static main(args) {
runBatch(new SynchAdministrativeMappingDataBatch(), args);
}
@Override
public void process(Object args) {
def retMap = synchAdminiMappingTo100()
synchToEs(retMap, "192.168.0.89")
// synchToEs(retMap, "192.168.0.52")
}
def synchAdminiMappingTo100(){
//删除旧的,再同步新的
ADMINISTRATIVE_MAPPING_COLL_100.remove(new BasicDBObject())
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(new BasicDBObject()).batchSize(1000)
def datas = []
def retMap = [:]
for(def d : dbcursor){
datas << d
retMap.put(d._id.toString(), d)
if (datas.size() >= 2000) {
ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
println datas.size()
datas.clear();
}
}
if (datas.size() >= 0) {
ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
datas.clear();
}
return retMap;
}
def synchToEs(devMap, host){
QueryBuilder queryBuilder = QueryBuilders.boolQuery()
ElasticUtils89.executeQuery(host, "administrative_mapping", queryBuilder, { String id, doc->
def mongoObj = devMap.get(id)
def updateFieldCount = 0
mongoObj.each{k,v->
if(!["id","_id"].contains(k)){
def val = doc.get(k)
if(val!=v){
println doc.name +" 修改 "+k+" "+ v+" "+id
// updateValMap.put(k,val)
updateFieldCount++
doc.put(k,v)
}
}
}
if(mongoObj && updateFieldCount){
//进行修改
return true
}else{
if(!mongoObj){
println "删除:"+id +" "+doc.name
ElasticUtils.delete(id, host, "administrative_mapping")
return false
}
}
})
}
}
@@ -28,8 +28,7 @@ class SynchDevDataBatch extends BaseBatch{
@Override
public void process(Object args) {
def devMap = synchDevTo100()
// synchDevToEs(devMap)
// synchDevEs(devMap, "192.168.0.89")
synchDevEs(devMap, "192.168.0.89")
synchDevEs(devMap, "192.168.0.52")
}