小区数据同步100 89 52

This commit is contained in:
2025-04-08 14:45:15 +08:00
parent d9a4f8de07
commit 076e724199
@@ -4,6 +4,7 @@ import com.fangjia.core.util.GeoUtils
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
@@ -15,7 +16,19 @@ import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.AMQP
import net.sf.json.JSONObject
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.*
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.bson.types.ObjectId
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.action.support.WriteRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.common.xcontent.XContentType
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
@@ -24,15 +37,17 @@ import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import com.fangjia.script.common.DataHandleUtils
/**
* 同步小区,商铺,写字楼到89,52上
*/
class SeaweedDataSynch extends BaseBatch{
static def targetUrl = "http://192.168.0.89:9200/seaweed/"
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL)
DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed_dishu", MONGO100_URL)
DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division_2023", MONGO100_URL)
DBCollection administrative_division_2022_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL)
@@ -136,43 +151,103 @@ class SeaweedDataSynch extends BaseBatch{
fullCityMap = CommonDatautils.queryFullNameCity()
mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
// priceMap = findAllDistrictSellPrice()
def citys = ["潮州"]//CommonDatautils.findAllCity()
priceMap = findAllDistrictSellPrice()
def citys = ["福州"]//CommonDatautils.findAllCity()
def index = 0
for(def c : citys){
index++
def dists = updating(c, "district")
updateEs(c, dists)
// updateEs(c, dists, "district", "192.168.0.89")
updateEs(c, dists, "district", "192.168.0.52")
// def offices = updating(c, "office")
// updateEs(c, offices, "office", "192.168.0.89")
// updateEs(c, offices, "office", "192.168.0.52")
updating(c, "office")
updating(c, "shop")
// def shops = updating(c, "shop")
// updateEs(c, shops, "shop", "192.168.0.89")
// updateEs(c, offices, "shop", "192.168.0.52")
println index +" "+ c + " 完成"
}
}
def updateEs(city, dists){
def updateEs(city, dists, cat, host){
def data = fullCityMap.get(city)
def distEsDatas = findEsSeaweedDatas(data.city)
def resultMap = [:]
for(def d : dists){
def esObj = distEsDatas.get(d._id.toString())
def map = [:]
d.each {k,v->
if(k!="_id"){
def esVal = esObj.get(k)
if(v!=esVal){
map.put(k, v)
def distEsDatas = findEsSeaweedDatas(data.city, cat, host)
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
def updateDatas = new CopyOnWriteArrayList()
def ids = new CopyOnWriteArrayList()
//更新89Es
for(def i=0;i<dists.size();i++){
final def d = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
ids << d._id.toString()
def esObj = distEsDatas.get(d._id.toString())
def map = updatingEs(d, esObj, host)
if(map){
updateDatas << map
}
} catch (Exception e) {
e.printStackTrace()
println d.city +" "+d.name
} finally {
latch.countDown()
}
}
}
if(map){
resultMap.put(d._id.toString(), map)
});
}
latch.await()
println "需更改:"+updateDatas.size()
updateSeaweedEs(updateDatas, host, "seaweed")
//将89Es中非正常小区删除
def esDelIds = []
distEsDatas.each{id,map->
if(!ids.contains(id)){
delEsData(id, host, "seaweed")
esDelIds << id
}
}
return resultMap
println "Es 删除 ${cat}类型:"+esDelIds.size()
//将100 mongodb中非正常小区删除
def mongoIds = findMongoDbDatas(cat, data.city)
mongoIds.each{id->
if(!ids.contains(id)){
SEAWEED_CONN.remove(new BasicDBObject("_id", new ObjectId(id)))
}
}
}
def updatingEs(d, esObj, host){
def map = [:]
def diffMap = [:]
d.each {k,v->
if(v && k!="_id"){
map.put(k, v)
}
if(esObj && k!="_id"){
def esVal = esObj.get(k)
if(esVal!=v){
diffMap.put(k,v)
}
}
}
if(!esObj || (esObj && diffMap)){
map.put("id", d._id.toString())
return map
}
return null
}
def updating(city, cat){
@@ -392,7 +467,7 @@ class SeaweedDataSynch extends BaseBatch{
map.avg_price = dataMap.avg_price
}
}
map.status = 0
// map.update_date = org.apache.commons.httpclient.util.DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T")
return map
}
@@ -448,12 +523,93 @@ class SeaweedDataSynch extends BaseBatch{
})
}
def findEsSeaweedDatas(city){
def findEsSeaweedDatas(city, cat, host){
def map = new HashMap()
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
ElasticUtils89.executeQuery("192.168.0.89", "seaweed", queryBuilder, { String id, doc->
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("category", cat))
ElasticUtils89.executeQuery(host, "seaweed", queryBuilder, { String id, doc->
map.put(id, doc)
})
return map;
}
def updateSeaweedEs(updateDatas, host, indexName){
BulkRequest request = new BulkRequest();
int processedCount = 0;
def client = null
if(host && host == "192.168.0.89"){
client = ElasticUtils89.getClient(host)
}else if(host && host == "192.168.0.52"){
client = ElasticUtils.getClient(host)
}
updateDatas.each{map->
request.add(new IndexRequest(indexName, "_doc", map.id).source(map, XContentType.JSON));
if (request.numberOfActions() >= 100) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents "+host
}
request = new BulkRequest();
}
}
if (request.numberOfActions() > 0) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents "+host
}
}
}
def delEsData(id, host, indexName){
def header = [:]
if(host && host == "192.168.0.89"){
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
}else if(host && host == "192.168.0.52"){
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:goojia123456".getBytes("UTF-8"))}".toString()
}
String seawaterUrl = "http://${host}:9200/${indexName}/_delete_by_query"
def body = """
{
"query": {
"bool": {
"must": [
{
"match": {
"id": "${id}"
}
}
]
}
}
}
"""
HttpRequest request = new HttpRequest()
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
println(res)
}
def findMongoDbDatas(cat, city){
def q = new BasicDBObject("category", cat).append("city", city)
def keys = new BasicDBObject()
keys.put("_id",1)
def dbcursor = SEAWEED_CONN.find(q, keys)
def mongoIds = []
for(def d : dbcursor){
mongoIds << d._id.toString()
}
return mongoIds
}
}