同步脚本修改
This commit is contained in:
+17
-13
@@ -1,11 +1,13 @@
|
|||||||
package com.fangjia.script.gj.data_handle
|
package com.fangjia.script.gj.data_handle
|
||||||
|
|
||||||
import com.fangjia.script.BaseBatch
|
import com.fangjia.script.BaseBatch
|
||||||
|
import com.fangjia.script.commData.CommonDatautils
|
||||||
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
||||||
import com.fangjia.script.util.ElasticUtils
|
import com.fangjia.script.util.ElasticUtils
|
||||||
import com.fangjia.script.util.MongoUtils
|
import com.fangjia.script.util.MongoUtils
|
||||||
import com.mongodb.BasicDBObject
|
import com.mongodb.BasicDBObject
|
||||||
import com.mongodb.DBCollection
|
import com.mongodb.DBCollection
|
||||||
|
import org.bson.types.ObjectId
|
||||||
import org.elasticsearch.index.query.QueryBuilder
|
import org.elasticsearch.index.query.QueryBuilder
|
||||||
import org.elasticsearch.index.query.QueryBuilders
|
import org.elasticsearch.index.query.QueryBuilders
|
||||||
|
|
||||||
@@ -23,12 +25,15 @@ class SynchAdministrativeMappingDataBatch extends BaseBatch{
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void process(Object args) {
|
public void process(Object args) {
|
||||||
|
def indexName = "administrative_mapping"
|
||||||
def retMap = synchAdminiMappingTo100()
|
def retMap = synchAdminiMappingTo100()
|
||||||
synchToEs(retMap, "192.168.0.89")
|
synchToEs(retMap, "192.168.0.89", indexName)
|
||||||
// synchToEs(retMap, "192.168.0.52")
|
synchToEs(retMap, "192.168.0.52", indexName)
|
||||||
|
println(" 运行完成")
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
def synchAdminiMappingTo100(){
|
def synchAdminiMappingTo100(city){
|
||||||
//删除旧的,再同步新的
|
//删除旧的,再同步新的
|
||||||
ADMINISTRATIVE_MAPPING_COLL_100.remove(new BasicDBObject())
|
ADMINISTRATIVE_MAPPING_COLL_100.remove(new BasicDBObject())
|
||||||
|
|
||||||
@@ -52,12 +57,11 @@ class SynchAdministrativeMappingDataBatch extends BaseBatch{
|
|||||||
return retMap;
|
return retMap;
|
||||||
}
|
}
|
||||||
|
|
||||||
def synchToEs(retMap, host){
|
def synchToEs(retMap, host, indexName){
|
||||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery()
|
QueryBuilder queryBuilder = QueryBuilders.boolQuery()
|
||||||
|
|
||||||
def oldIds = new CopyOnWriteArrayList()
|
def oldIds = new CopyOnWriteArrayList()
|
||||||
|
|
||||||
ElasticUtils89.executeQuery(host, "administrative_mapping", queryBuilder, { String id, doc->
|
ElasticUtils89.queryUpdate(host, indexName, queryBuilder, { String id, doc->
|
||||||
oldIds << id
|
oldIds << id
|
||||||
def mongoObj = retMap.get(id)
|
def mongoObj = retMap.get(id)
|
||||||
def updateFieldCount = 0
|
def updateFieldCount = 0
|
||||||
@@ -65,8 +69,7 @@ class SynchAdministrativeMappingDataBatch extends BaseBatch{
|
|||||||
if(!["id","_id","c_date","m_date","polyline_source","operator","city_simple_pinyin","prefecture_level_city"].contains(k)){
|
if(!["id","_id","c_date","m_date","polyline_source","operator","city_simple_pinyin","prefecture_level_city"].contains(k)){
|
||||||
def val = doc.get(k)
|
def val = doc.get(k)
|
||||||
if(val!=v){
|
if(val!=v){
|
||||||
// updateValMap.put(k,val)
|
if(k!="shape" || (k=="shape" && v.type!="MultiPolygon")){
|
||||||
if(v.type!="MultiPolygon"){
|
|
||||||
println k +" 修改 "+id
|
println k +" 修改 "+id
|
||||||
updateFieldCount++
|
updateFieldCount++
|
||||||
doc.put(k,v)
|
doc.put(k,v)
|
||||||
@@ -80,14 +83,14 @@ class SynchAdministrativeMappingDataBatch extends BaseBatch{
|
|||||||
}else{
|
}else{
|
||||||
if(!mongoObj){
|
if(!mongoObj){
|
||||||
println "删除:"+id +" "+doc.name
|
println "删除:"+id +" "+doc.name
|
||||||
ElasticUtils.delete(id, host, "administrative_mapping")
|
ElasticUtils.delete(id, host, indexName)
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
||||||
def addEsDatas = []
|
def addEsDatasMap = [:]
|
||||||
retMap.each{id,map->
|
retMap.each{id,map->
|
||||||
if(!oldIds.contains(id)){
|
if(!oldIds.contains(id)){
|
||||||
println "需新增:"+id
|
println "需新增:"+id
|
||||||
@@ -95,11 +98,12 @@ class SynchAdministrativeMappingDataBatch extends BaseBatch{
|
|||||||
map.remove("m_date")
|
map.remove("m_date")
|
||||||
map.remove("polyline_source")
|
map.remove("polyline_source")
|
||||||
map.remove("operator")
|
map.remove("operator")
|
||||||
addEsDatas << map
|
map.remove("_id")
|
||||||
|
addEsDatasMap.put(id, map)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if(addEsDatas){
|
if(addEsDatasMap){
|
||||||
ElasticUtils89.updateEs(addEsDatas, host, "administrative_mapping")
|
ElasticUtils89.updateEs(addEsDatasMap, host, indexName)
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
+30
-6
@@ -10,6 +10,7 @@ import org.elasticsearch.index.query.QueryBuilder
|
|||||||
import org.elasticsearch.index.query.QueryBuilders
|
import org.elasticsearch.index.query.QueryBuilders
|
||||||
|
|
||||||
import java.util.concurrent.ConcurrentHashMap
|
import java.util.concurrent.ConcurrentHashMap
|
||||||
|
import java.util.concurrent.CopyOnWriteArrayList
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 将43上的开发区同步到mongo-100, es-89, es-52
|
* 将43上的开发区同步到mongo-100, es-89, es-52
|
||||||
@@ -27,9 +28,10 @@ class SynchDevDataBatch extends BaseBatch{
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void process(Object args) {
|
public void process(Object args) {
|
||||||
|
def indexName = "development_zone"
|
||||||
def devMap = synchDevTo100()
|
def devMap = synchDevTo100()
|
||||||
synchDevEs(devMap, "192.168.0.89")
|
synchDevEs(devMap, "192.168.0.89", indexName)
|
||||||
synchDevEs(devMap, "192.168.0.52")
|
synchDevEs(devMap, "192.168.0.52", indexName)
|
||||||
}
|
}
|
||||||
|
|
||||||
def synchDevTo100(){
|
def synchDevTo100(){
|
||||||
@@ -81,10 +83,12 @@ class SynchDevDataBatch extends BaseBatch{
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def synchDevEs(devMap, host){
|
def synchDevEs(devMap, host, indexName){
|
||||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery()
|
QueryBuilder queryBuilder = QueryBuilders.boolQuery()
|
||||||
|
def oldIds = new CopyOnWriteArrayList()
|
||||||
|
|
||||||
ElasticUtils89.executeQuery(host, "development_zone", queryBuilder, { String id, doc->
|
ElasticUtils89.executeQuery(host, indexName, queryBuilder, { String id, doc->
|
||||||
|
oldIds << id
|
||||||
def mongoObj = devMap.get(id)
|
def mongoObj = devMap.get(id)
|
||||||
def updateFieldCount = 0
|
def updateFieldCount = 0
|
||||||
mongoObj.each{k,v->
|
mongoObj.each{k,v->
|
||||||
@@ -92,7 +96,6 @@ class SynchDevDataBatch extends BaseBatch{
|
|||||||
def val = doc.get(k)
|
def val = doc.get(k)
|
||||||
if(val!=v){
|
if(val!=v){
|
||||||
println doc.name +" 修改 "+k+" "+ v+" "+id
|
println doc.name +" 修改 "+k+" "+ v+" "+id
|
||||||
// updateValMap.put(k,val)
|
|
||||||
updateFieldCount++
|
updateFieldCount++
|
||||||
doc.put(k,v)
|
doc.put(k,v)
|
||||||
}
|
}
|
||||||
@@ -104,11 +107,32 @@ class SynchDevDataBatch extends BaseBatch{
|
|||||||
}else{
|
}else{
|
||||||
if(!mongoObj){
|
if(!mongoObj){
|
||||||
println "删除:"+id +" "+doc.name
|
println "删除:"+id +" "+doc.name
|
||||||
ElasticUtils.delete(id, host, "development_zone")
|
ElasticUtils.delete(id, host, indexName)
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
def addEsDatasMap = [:]
|
||||||
|
devMap.each{id,map->
|
||||||
|
if(!oldIds.contains(id)){
|
||||||
|
println "需新增:"+id
|
||||||
|
map.remove("create_time")
|
||||||
|
map.remove("build_date")
|
||||||
|
map.remove("bd_shape")
|
||||||
|
map.remove("img_url")
|
||||||
|
map.remove("operator")
|
||||||
|
map.remove("operate_result")
|
||||||
|
map.remove("m_date")
|
||||||
|
map.remove("township")
|
||||||
|
map.remove("block")
|
||||||
|
map.remove("_id")
|
||||||
|
addEsDatasMap.put(id, map)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if(addEsDatasMap){
|
||||||
|
ElasticUtils89.updateEs(addEsDatasMap, host, indexName)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
-2
@@ -78,8 +78,6 @@ class SeaweedDataSynch extends BaseBatch{
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void process(Object args) {
|
public void process(Object args) {
|
||||||
|
|
||||||
|
|
||||||
run()
|
run()
|
||||||
// println "全国district, shop, office同步完成"
|
// println "全国district, shop, office同步完成"
|
||||||
//民政部编码上传
|
//民政部编码上传
|
||||||
|
|||||||
Reference in New Issue
Block a user