From 076e72419925a658c4781bf9c636d6c602efb84f Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 8 Apr 2025 14:45:15 +0800 Subject: [PATCH] =?UTF-8?q?=E5=B0=8F=E5=8C=BA=E6=95=B0=E6=8D=AE=E5=90=8C?= =?UTF-8?q?=E6=AD=A5100=2089=2052?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/gj/xiamen/SeaweedDataSynch.groovy | 208 +++++++++++++++--- 1 file changed, 182 insertions(+), 26 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy index b5f10fe8d..65bbe11f6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy @@ -4,6 +4,7 @@ import com.fangjia.core.util.GeoUtils import com.fangjia.core.util.HttpRequest import com.fangjia.script.BaseBatch import com.fangjia.script.commData.CommonDatautils + import com.fangjia.script.util.DateUtil import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DbUtils @@ -15,7 +16,19 @@ import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.rabbitmq.client.AMQP import net.sf.json.JSONObject + +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.util.* +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + import org.bson.types.ObjectId +import org.elasticsearch.action.bulk.BulkRequest +import org.elasticsearch.action.bulk.BulkResponse +import org.elasticsearch.action.index.IndexRequest +import org.elasticsearch.action.support.WriteRequest +import org.elasticsearch.client.RequestOptions +import org.elasticsearch.common.xcontent.XContentType import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilders @@ -24,15 +37,17 @@ import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicLong -import com.fangjia.script.common.DataHandleUtils +/** + * 同步小区,商铺,写字楼到89,52上 + */ class SeaweedDataSynch extends BaseBatch{ static def targetUrl = "http://192.168.0.89:9200/seaweed/" public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; - DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL) + DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed_dishu", MONGO100_URL) DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division_2023", MONGO100_URL) DBCollection administrative_division_2022_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL) @@ -136,43 +151,103 @@ class SeaweedDataSynch extends BaseBatch{ fullCityMap = CommonDatautils.queryFullNameCity() mcaDistMap = CommonDatautils.queryDistNameCityByMapping() -// priceMap = findAllDistrictSellPrice() - def citys = ["潮州"]//CommonDatautils.findAllCity() + priceMap = findAllDistrictSellPrice() + def citys = ["福州"]//CommonDatautils.findAllCity() def index = 0 for(def c : citys){ index++ def dists = updating(c, "district") - updateEs(c, dists) +// updateEs(c, dists, "district", "192.168.0.89") + updateEs(c, dists, "district", "192.168.0.52") +// def offices = updating(c, "office") +// updateEs(c, offices, "office", "192.168.0.89") +// updateEs(c, offices, "office", "192.168.0.52") - updating(c, "office") - updating(c, "shop") +// def shops = updating(c, "shop") +// updateEs(c, shops, "shop", "192.168.0.89") +// updateEs(c, offices, "shop", "192.168.0.52") println index +" "+ c + " 完成" } } - def updateEs(city, dists){ + def updateEs(city, dists, cat, host){ def data = fullCityMap.get(city) - def distEsDatas = findEsSeaweedDatas(data.city) - def resultMap = [:] - for(def d : dists){ - def esObj = distEsDatas.get(d._id.toString()) - def map = [:] - d.each {k,v-> - if(k!="_id"){ - def esVal = esObj.get(k) - if(v!=esVal){ - map.put(k, v) + def distEsDatas = findEsSeaweedDatas(data.city, cat, host) + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(dists.size()) + def updateDatas = new CopyOnWriteArrayList() + def ids = new CopyOnWriteArrayList() + //更新89Es + for(def i=0;i + if(!ids.contains(id)){ + delEsData(id, host, "seaweed") + esDelIds << id } } - return resultMap + println "Es 删除 ${cat}类型:"+esDelIds.size() + //将100 mongodb中非正常小区删除 + def mongoIds = findMongoDbDatas(cat, data.city) + + mongoIds.each{id-> + if(!ids.contains(id)){ + SEAWEED_CONN.remove(new BasicDBObject("_id", new ObjectId(id))) + } + } + + } + + def updatingEs(d, esObj, host){ + def map = [:] + def diffMap = [:] + d.each {k,v-> + if(v && k!="_id"){ + map.put(k, v) + } + if(esObj && k!="_id"){ + def esVal = esObj.get(k) + if(esVal!=v){ + diffMap.put(k,v) + } + } + } + if(!esObj || (esObj && diffMap)){ + map.put("id", d._id.toString()) + return map + } + return null } def updating(city, cat){ @@ -392,7 +467,7 @@ class SeaweedDataSynch extends BaseBatch{ map.avg_price = dataMap.avg_price } } - + map.status = 0 // map.update_date = org.apache.commons.httpclient.util.DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T") return map } @@ -448,12 +523,93 @@ class SeaweedDataSynch extends BaseBatch{ }) } - def findEsSeaweedDatas(city){ + def findEsSeaweedDatas(city, cat, host){ def map = new HashMap() - QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)) - ElasticUtils89.executeQuery("192.168.0.89", "seaweed", queryBuilder, { String id, doc-> + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("category", cat)) + ElasticUtils89.executeQuery(host, "seaweed", queryBuilder, { String id, doc-> map.put(id, doc) }) return map; } + + def updateSeaweedEs(updateDatas, host, indexName){ + BulkRequest request = new BulkRequest(); + int processedCount = 0; + def client = null + if(host && host == "192.168.0.89"){ + client = ElasticUtils89.getClient(host) + }else if(host && host == "192.168.0.52"){ + client = ElasticUtils.getClient(host) + } + + updateDatas.each{map-> + request.add(new IndexRequest(indexName, "_doc", map.id).source(map, XContentType.JSON)); + if (request.numberOfActions() >= 100) { + request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); + if (bulkResponse.hasFailures()) { + // 处理错误 + println "Bulk request has failures: " + bulkResponse.buildFailureMessage() + } else { + processedCount += request.numberOfActions(); + println "Processed " + processedCount + " documents "+host + } + request = new BulkRequest(); + } + } + + if (request.numberOfActions() > 0) { + request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); + if (bulkResponse.hasFailures()) { + // 处理错误 + println "Bulk request has failures: " + bulkResponse.buildFailureMessage() + } else { + processedCount += request.numberOfActions(); + println "Processed " + processedCount + " documents "+host + } + } + } + + def delEsData(id, host, indexName){ + def header = [:] + if(host && host == "192.168.0.89"){ + header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString() + }else if(host && host == "192.168.0.52"){ + header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:goojia123456".getBytes("UTF-8"))}".toString() + } + String seawaterUrl = "http://${host}:9200/${indexName}/_delete_by_query" + def body = """ + { + "query": { + "bool": { + "must": [ + { + "match": { + "id": "${id}" + } + } + ] + } + } + } + """ + HttpRequest request = new HttpRequest() + String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8") + println(res) + } + + def findMongoDbDatas(cat, city){ + def q = new BasicDBObject("category", cat).append("city", city) + def keys = new BasicDBObject() + keys.put("_id",1) + def dbcursor = SEAWEED_CONN.find(q, keys) + def mongoIds = [] + for(def d : dbcursor){ + mongoIds << d._id.toString() + } + return mongoIds + } + + }