同步脚本修改

This commit is contained in:
2025-04-09 17:44:28 +08:00
parent d1a821af0c
commit 31cd461b8a
3 changed files with 47 additions and 21 deletions
@@ -1,11 +1,13 @@
package com.fangjia.script.gj.data_handle package com.fangjia.script.gj.data_handle
import com.fangjia.script.BaseBatch import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.xiamen.ElasticUtils89 import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.DBCollection import com.mongodb.DBCollection
import org.bson.types.ObjectId
import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders import org.elasticsearch.index.query.QueryBuilders
@@ -23,12 +25,15 @@ class SynchAdministrativeMappingDataBatch extends BaseBatch{
@Override @Override
public void process(Object args) { public void process(Object args) {
def indexName = "administrative_mapping"
def retMap = synchAdminiMappingTo100() def retMap = synchAdminiMappingTo100()
synchToEs(retMap, "192.168.0.89") synchToEs(retMap, "192.168.0.89", indexName)
// synchToEs(retMap, "192.168.0.52") synchToEs(retMap, "192.168.0.52", indexName)
println(" 运行完成")
} }
def synchAdminiMappingTo100(){ def synchAdminiMappingTo100(city){
//删除旧的,再同步新的 //删除旧的,再同步新的
ADMINISTRATIVE_MAPPING_COLL_100.remove(new BasicDBObject()) ADMINISTRATIVE_MAPPING_COLL_100.remove(new BasicDBObject())
@@ -52,12 +57,11 @@ class SynchAdministrativeMappingDataBatch extends BaseBatch{
return retMap; return retMap;
} }
def synchToEs(retMap, host){ def synchToEs(retMap, host, indexName){
QueryBuilder queryBuilder = QueryBuilders.boolQuery() QueryBuilder queryBuilder = QueryBuilders.boolQuery()
def oldIds = new CopyOnWriteArrayList() def oldIds = new CopyOnWriteArrayList()
ElasticUtils89.executeQuery(host, "administrative_mapping", queryBuilder, { String id, doc-> ElasticUtils89.queryUpdate(host, indexName, queryBuilder, { String id, doc->
oldIds << id oldIds << id
def mongoObj = retMap.get(id) def mongoObj = retMap.get(id)
def updateFieldCount = 0 def updateFieldCount = 0
@@ -65,8 +69,7 @@ class SynchAdministrativeMappingDataBatch extends BaseBatch{
if(!["id","_id","c_date","m_date","polyline_source","operator","city_simple_pinyin","prefecture_level_city"].contains(k)){ if(!["id","_id","c_date","m_date","polyline_source","operator","city_simple_pinyin","prefecture_level_city"].contains(k)){
def val = doc.get(k) def val = doc.get(k)
if(val!=v){ if(val!=v){
// updateValMap.put(k,val) if(k!="shape" || (k=="shape" && v.type!="MultiPolygon")){
if(v.type!="MultiPolygon"){
println k +" 修改 "+id println k +" 修改 "+id
updateFieldCount++ updateFieldCount++
doc.put(k,v) doc.put(k,v)
@@ -80,14 +83,14 @@ class SynchAdministrativeMappingDataBatch extends BaseBatch{
}else{ }else{
if(!mongoObj){ if(!mongoObj){
println "删除:"+id +" "+doc.name println "删除:"+id +" "+doc.name
ElasticUtils.delete(id, host, "administrative_mapping") ElasticUtils.delete(id, host, indexName)
return false return false
} }
} }
}) })
def addEsDatas = [] def addEsDatasMap = [:]
retMap.each{id,map-> retMap.each{id,map->
if(!oldIds.contains(id)){ if(!oldIds.contains(id)){
println "需新增:"+id println "需新增:"+id
@@ -95,11 +98,12 @@ class SynchAdministrativeMappingDataBatch extends BaseBatch{
map.remove("m_date") map.remove("m_date")
map.remove("polyline_source") map.remove("polyline_source")
map.remove("operator") map.remove("operator")
addEsDatas << map map.remove("_id")
addEsDatasMap.put(id, map)
} }
} }
if(addEsDatas){ if(addEsDatasMap){
ElasticUtils89.updateEs(addEsDatas, host, "administrative_mapping") ElasticUtils89.updateEs(addEsDatasMap, host, indexName)
} }
} }
@@ -10,6 +10,7 @@ import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders import org.elasticsearch.index.query.QueryBuilders
import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArrayList
/** /**
* 将43上的开发区同步到mongo-100, es-89, es-52 * 将43上的开发区同步到mongo-100, es-89, es-52
@@ -27,9 +28,10 @@ class SynchDevDataBatch extends BaseBatch{
@Override @Override
public void process(Object args) { public void process(Object args) {
def indexName = "development_zone"
def devMap = synchDevTo100() def devMap = synchDevTo100()
synchDevEs(devMap, "192.168.0.89") synchDevEs(devMap, "192.168.0.89", indexName)
synchDevEs(devMap, "192.168.0.52") synchDevEs(devMap, "192.168.0.52", indexName)
} }
def synchDevTo100(){ def synchDevTo100(){
@@ -81,10 +83,12 @@ class SynchDevDataBatch extends BaseBatch{
} }
def synchDevEs(devMap, host){ def synchDevEs(devMap, host, indexName){
QueryBuilder queryBuilder = QueryBuilders.boolQuery() QueryBuilder queryBuilder = QueryBuilders.boolQuery()
def oldIds = new CopyOnWriteArrayList()
ElasticUtils89.executeQuery(host, "development_zone", queryBuilder, { String id, doc-> ElasticUtils89.executeQuery(host, indexName, queryBuilder, { String id, doc->
oldIds << id
def mongoObj = devMap.get(id) def mongoObj = devMap.get(id)
def updateFieldCount = 0 def updateFieldCount = 0
mongoObj.each{k,v-> mongoObj.each{k,v->
@@ -92,7 +96,6 @@ class SynchDevDataBatch extends BaseBatch{
def val = doc.get(k) def val = doc.get(k)
if(val!=v){ if(val!=v){
println doc.name +" 修改 "+k+" "+ v+" "+id println doc.name +" 修改 "+k+" "+ v+" "+id
// updateValMap.put(k,val)
updateFieldCount++ updateFieldCount++
doc.put(k,v) doc.put(k,v)
} }
@@ -104,11 +107,32 @@ class SynchDevDataBatch extends BaseBatch{
}else{ }else{
if(!mongoObj){ if(!mongoObj){
println "删除:"+id +" "+doc.name println "删除:"+id +" "+doc.name
ElasticUtils.delete(id, host, "development_zone") ElasticUtils.delete(id, host, indexName)
return false return false
} }
} }
}) })
def addEsDatasMap = [:]
devMap.each{id,map->
if(!oldIds.contains(id)){
println "需新增:"+id
map.remove("create_time")
map.remove("build_date")
map.remove("bd_shape")
map.remove("img_url")
map.remove("operator")
map.remove("operate_result")
map.remove("m_date")
map.remove("township")
map.remove("block")
map.remove("_id")
addEsDatasMap.put(id, map)
}
}
if(addEsDatasMap){
ElasticUtils89.updateEs(addEsDatasMap, host, indexName)
}
} }
@@ -78,8 +78,6 @@ class SeaweedDataSynch extends BaseBatch{
@Override @Override
public void process(Object args) { public void process(Object args) {
run() run()
// println "全国district, shop, office同步完成" // println "全国district, shop, office同步完成"
//民政部编码上传 //民政部编码上传