diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/SynchAdministrativeMappingDataBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/SynchAdministrativeMappingDataBatch.groovy index 00b912a8c..ddb9616bd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/SynchAdministrativeMappingDataBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/SynchAdministrativeMappingDataBatch.groovy @@ -1,11 +1,13 @@ package com.fangjia.script.gj.data_handle import com.fangjia.script.BaseBatch +import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.xiamen.ElasticUtils89 import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import org.bson.types.ObjectId import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilders @@ -23,12 +25,15 @@ class SynchAdministrativeMappingDataBatch extends BaseBatch{ @Override public void process(Object args) { + def indexName = "administrative_mapping" def retMap = synchAdminiMappingTo100() - synchToEs(retMap, "192.168.0.89") -// synchToEs(retMap, "192.168.0.52") + synchToEs(retMap, "192.168.0.89", indexName) + synchToEs(retMap, "192.168.0.52", indexName) + println(" 运行完成") + } - def synchAdminiMappingTo100(){ + def synchAdminiMappingTo100(city){ //删除旧的,再同步新的 ADMINISTRATIVE_MAPPING_COLL_100.remove(new BasicDBObject()) @@ -52,12 +57,11 @@ class SynchAdministrativeMappingDataBatch extends BaseBatch{ return retMap; } - def synchToEs(retMap, host){ + def synchToEs(retMap, host, indexName){ QueryBuilder queryBuilder = QueryBuilders.boolQuery() - def oldIds = new CopyOnWriteArrayList() - ElasticUtils89.executeQuery(host, "administrative_mapping", queryBuilder, { String id, doc-> + ElasticUtils89.queryUpdate(host, indexName, queryBuilder, { String id, doc-> oldIds << id def mongoObj = retMap.get(id) def updateFieldCount = 0 @@ -65,8 +69,7 @@ class SynchAdministrativeMappingDataBatch extends BaseBatch{ if(!["id","_id","c_date","m_date","polyline_source","operator","city_simple_pinyin","prefecture_level_city"].contains(k)){ def val = doc.get(k) if(val!=v){ -// updateValMap.put(k,val) - if(v.type!="MultiPolygon"){ + if(k!="shape" || (k=="shape" && v.type!="MultiPolygon")){ println k +" 修改 "+id updateFieldCount++ doc.put(k,v) @@ -80,14 +83,14 @@ class SynchAdministrativeMappingDataBatch extends BaseBatch{ }else{ if(!mongoObj){ println "删除:"+id +" "+doc.name - ElasticUtils.delete(id, host, "administrative_mapping") + ElasticUtils.delete(id, host, indexName) return false } } }) - def addEsDatas = [] + def addEsDatasMap = [:] retMap.each{id,map-> if(!oldIds.contains(id)){ println "需新增:"+id @@ -95,11 +98,12 @@ class SynchAdministrativeMappingDataBatch extends BaseBatch{ map.remove("m_date") map.remove("polyline_source") map.remove("operator") - addEsDatas << map + map.remove("_id") + addEsDatasMap.put(id, map) } } - if(addEsDatas){ - ElasticUtils89.updateEs(addEsDatas, host, "administrative_mapping") + if(addEsDatasMap){ + ElasticUtils89.updateEs(addEsDatasMap, host, indexName) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/SynchDevDataBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/SynchDevDataBatch.groovy index 893da6125..ea5a16f8e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/SynchDevDataBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/SynchDevDataBatch.groovy @@ -10,6 +10,7 @@ import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilders import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CopyOnWriteArrayList /** * 将43上的开发区同步到mongo-100, es-89, es-52 @@ -27,9 +28,10 @@ class SynchDevDataBatch extends BaseBatch{ @Override public void process(Object args) { + def indexName = "development_zone" def devMap = synchDevTo100() - synchDevEs(devMap, "192.168.0.89") - synchDevEs(devMap, "192.168.0.52") + synchDevEs(devMap, "192.168.0.89", indexName) + synchDevEs(devMap, "192.168.0.52", indexName) } def synchDevTo100(){ @@ -81,10 +83,12 @@ class SynchDevDataBatch extends BaseBatch{ } - def synchDevEs(devMap, host){ + def synchDevEs(devMap, host, indexName){ QueryBuilder queryBuilder = QueryBuilders.boolQuery() + def oldIds = new CopyOnWriteArrayList() - ElasticUtils89.executeQuery(host, "development_zone", queryBuilder, { String id, doc-> + ElasticUtils89.executeQuery(host, indexName, queryBuilder, { String id, doc-> + oldIds << id def mongoObj = devMap.get(id) def updateFieldCount = 0 mongoObj.each{k,v-> @@ -92,7 +96,6 @@ class SynchDevDataBatch extends BaseBatch{ def val = doc.get(k) if(val!=v){ println doc.name +" 修改 "+k+" "+ v+" "+id -// updateValMap.put(k,val) updateFieldCount++ doc.put(k,v) } @@ -104,11 +107,32 @@ class SynchDevDataBatch extends BaseBatch{ }else{ if(!mongoObj){ println "删除:"+id +" "+doc.name - ElasticUtils.delete(id, host, "development_zone") + ElasticUtils.delete(id, host, indexName) return false } } }) + + def addEsDatasMap = [:] + devMap.each{id,map-> + if(!oldIds.contains(id)){ + println "需新增:"+id + map.remove("create_time") + map.remove("build_date") + map.remove("bd_shape") + map.remove("img_url") + map.remove("operator") + map.remove("operate_result") + map.remove("m_date") + map.remove("township") + map.remove("block") + map.remove("_id") + addEsDatasMap.put(id, map) + } + } + if(addEsDatasMap){ + ElasticUtils89.updateEs(addEsDatasMap, host, indexName) + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy index 0955b9921..6f07d1516 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy @@ -78,8 +78,6 @@ class SeaweedDataSynch extends BaseBatch{ @Override public void process(Object args) { - - run() // println "全国district, shop, office同步完成" //民政部编码上传