diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/SynchDevDataBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/SynchDevDataBatch.groovy new file mode 100644 index 000000000..dc96e2246 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/SynchDevDataBatch.groovy @@ -0,0 +1,119 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.BaseBatch +import com.fangjia.script.gj.xiamen.ElasticUtils89 +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +import java.util.concurrent.ConcurrentHashMap + +/** + * 将43上的开发区同步到mongo-100, es-89, es-52 + */ +class SynchDevDataBatch extends BaseBatch{ + + def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; + DBCollection development_zone_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL) + + DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com") + + static main(args) { + runBatch(new SynchDevDataBatch(), args); + } + + @Override + public void process(Object args) { + def devMap = synchDevTo100() +// synchDevToEs(devMap) +// synchDevEs(devMap, "192.168.0.89") + synchDevEs(devMap, "192.168.0.52") + } + + def synchDevTo100(){ + def dbcursor = DEVELOPMENT_ZONE_43_COLL.find() + def rets = [] + def devMap = [:] + for(def r : dbcursor){ + rets << r + devMap.put(r._id.toString(), r) + } + //删除旧的,再同步新的 + development_zone_CONN.remove(new BasicDBObject()) + development_zone_CONN.insert(rets) + return devMap; + } + + def synchDevToEs(devMap){ +// def esDevs = findEsDevDatas() +// def ids = [] +// for(def dev : devDatas){ +// ids << dev._id.toString() +// def esObj = esDevs.get(dev._id.toString()) +// def updateValMap = [:] +// if(esObj){ +// dev.each{k,v-> +// if(k!="id" && k!="_id"){ +// def val = dev.get(k) +// if(val!=v){ +// updateValMap.put(k,val) +// } +// } +// } +// } +// if(!esObj || (esObj && updateValMap)){ +// ElasticUtils.modify(dev._id.toString(), dev, "192.168.0.89", "development_zone") +// ElasticUtils.modify(dev._id.toString(), dev, "192.168.0.52", "development_zone") +// } +// } +// +// def esDelIds = [] +// esDevs.each{id,map-> +// if(!ids.contains(id)){ +// ElasticUtils.delete(id, "192.168.0.89", "development_zone") +// ElasticUtils.delete(id, "192.168.0.52", "development_zone") +// esDelIds << id +// } +// } +// println "Es删除开发区数量:"+esDelIds.size() + } + + + def synchDevEs(devMap, host){ + QueryBuilder queryBuilder = QueryBuilders.boolQuery() + + ElasticUtils89.executeQuery(host, "development_zone", queryBuilder, { String id, doc-> + def mongoObj = devMap.get(id) + def updateFieldCount = 0 + mongoObj.each{k,v-> + if(!["id","_id" ,"build_date" ,"create_time","bd_shape","img_url","operator","operate_result","m_date","township","block"].contains(k)){ + def val = doc.get(k) + if(val!=v){ + println doc.name +" 修改 "+k+" "+ v+" "+id +// updateValMap.put(k,val) + updateFieldCount++ + doc.put(k,v) + } + } + } + if(mongoObj && updateFieldCount){ + //进行修改 + return true + }else{ + if(!mongoObj){ + println "删除:"+id +" "+doc.name + ElasticUtils.delete(id, host, "development_zone") + return false + } + } + }) + } + + + + + +}