小区数据同步100 89 52
This commit is contained in:
+182
-26
@@ -4,6 +4,7 @@ import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
|
||||
import com.fangjia.script.util.DateUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
@@ -15,7 +16,19 @@ import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.AMQP
|
||||
import net.sf.json.JSONObject
|
||||
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.*
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
import org.elasticsearch.action.index.IndexRequest
|
||||
import org.elasticsearch.action.support.WriteRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.common.xcontent.XContentType
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
@@ -24,15 +37,17 @@ import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
|
||||
/**
|
||||
* 同步小区,商铺,写字楼到89,52上
|
||||
*/
|
||||
class SeaweedDataSynch extends BaseBatch{
|
||||
|
||||
static def targetUrl = "http://192.168.0.89:9200/seaweed/"
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL)
|
||||
DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed_dishu", MONGO100_URL)
|
||||
DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division_2023", MONGO100_URL)
|
||||
DBCollection administrative_division_2022_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL)
|
||||
|
||||
@@ -136,43 +151,103 @@ class SeaweedDataSynch extends BaseBatch{
|
||||
fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
|
||||
|
||||
// priceMap = findAllDistrictSellPrice()
|
||||
def citys = ["潮州"]//CommonDatautils.findAllCity()
|
||||
priceMap = findAllDistrictSellPrice()
|
||||
def citys = ["福州"]//CommonDatautils.findAllCity()
|
||||
def index = 0
|
||||
for(def c : citys){
|
||||
index++
|
||||
|
||||
def dists = updating(c, "district")
|
||||
updateEs(c, dists)
|
||||
// updateEs(c, dists, "district", "192.168.0.89")
|
||||
updateEs(c, dists, "district", "192.168.0.52")
|
||||
|
||||
// def offices = updating(c, "office")
|
||||
// updateEs(c, offices, "office", "192.168.0.89")
|
||||
// updateEs(c, offices, "office", "192.168.0.52")
|
||||
|
||||
updating(c, "office")
|
||||
updating(c, "shop")
|
||||
// def shops = updating(c, "shop")
|
||||
// updateEs(c, shops, "shop", "192.168.0.89")
|
||||
// updateEs(c, offices, "shop", "192.168.0.52")
|
||||
|
||||
println index +" "+ c + " 完成"
|
||||
}
|
||||
}
|
||||
|
||||
def updateEs(city, dists){
|
||||
def updateEs(city, dists, cat, host){
|
||||
def data = fullCityMap.get(city)
|
||||
def distEsDatas = findEsSeaweedDatas(data.city)
|
||||
def resultMap = [:]
|
||||
for(def d : dists){
|
||||
def esObj = distEsDatas.get(d._id.toString())
|
||||
def map = [:]
|
||||
d.each {k,v->
|
||||
if(k!="_id"){
|
||||
def esVal = esObj.get(k)
|
||||
if(v!=esVal){
|
||||
map.put(k, v)
|
||||
def distEsDatas = findEsSeaweedDatas(data.city, cat, host)
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
def updateDatas = new CopyOnWriteArrayList()
|
||||
def ids = new CopyOnWriteArrayList()
|
||||
//更新89Es
|
||||
for(def i=0;i<dists.size();i++){
|
||||
final def d = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
ids << d._id.toString()
|
||||
def esObj = distEsDatas.get(d._id.toString())
|
||||
def map = updatingEs(d, esObj, host)
|
||||
if(map){
|
||||
updateDatas << map
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
println d.city +" "+d.name
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
}
|
||||
if(map){
|
||||
resultMap.put(d._id.toString(), map)
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
|
||||
println "需更改:"+updateDatas.size()
|
||||
|
||||
|
||||
updateSeaweedEs(updateDatas, host, "seaweed")
|
||||
|
||||
//将89Es中非正常小区删除
|
||||
def esDelIds = []
|
||||
distEsDatas.each{id,map->
|
||||
if(!ids.contains(id)){
|
||||
delEsData(id, host, "seaweed")
|
||||
esDelIds << id
|
||||
}
|
||||
}
|
||||
return resultMap
|
||||
println "Es 删除 ${cat}类型:"+esDelIds.size()
|
||||
//将100 mongodb中非正常小区删除
|
||||
def mongoIds = findMongoDbDatas(cat, data.city)
|
||||
|
||||
mongoIds.each{id->
|
||||
if(!ids.contains(id)){
|
||||
SEAWEED_CONN.remove(new BasicDBObject("_id", new ObjectId(id)))
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def updatingEs(d, esObj, host){
|
||||
def map = [:]
|
||||
def diffMap = [:]
|
||||
d.each {k,v->
|
||||
if(v && k!="_id"){
|
||||
map.put(k, v)
|
||||
}
|
||||
if(esObj && k!="_id"){
|
||||
def esVal = esObj.get(k)
|
||||
if(esVal!=v){
|
||||
diffMap.put(k,v)
|
||||
}
|
||||
}
|
||||
}
|
||||
if(!esObj || (esObj && diffMap)){
|
||||
map.put("id", d._id.toString())
|
||||
return map
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def updating(city, cat){
|
||||
@@ -392,7 +467,7 @@ class SeaweedDataSynch extends BaseBatch{
|
||||
map.avg_price = dataMap.avg_price
|
||||
}
|
||||
}
|
||||
|
||||
map.status = 0
|
||||
// map.update_date = org.apache.commons.httpclient.util.DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T")
|
||||
return map
|
||||
}
|
||||
@@ -448,12 +523,93 @@ class SeaweedDataSynch extends BaseBatch{
|
||||
})
|
||||
}
|
||||
|
||||
def findEsSeaweedDatas(city){
|
||||
def findEsSeaweedDatas(city, cat, host){
|
||||
def map = new HashMap()
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
|
||||
ElasticUtils89.executeQuery("192.168.0.89", "seaweed", queryBuilder, { String id, doc->
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("category", cat))
|
||||
ElasticUtils89.executeQuery(host, "seaweed", queryBuilder, { String id, doc->
|
||||
map.put(id, doc)
|
||||
})
|
||||
return map;
|
||||
}
|
||||
|
||||
def updateSeaweedEs(updateDatas, host, indexName){
|
||||
BulkRequest request = new BulkRequest();
|
||||
int processedCount = 0;
|
||||
def client = null
|
||||
if(host && host == "192.168.0.89"){
|
||||
client = ElasticUtils89.getClient(host)
|
||||
}else if(host && host == "192.168.0.52"){
|
||||
client = ElasticUtils.getClient(host)
|
||||
}
|
||||
|
||||
updateDatas.each{map->
|
||||
request.add(new IndexRequest(indexName, "_doc", map.id).source(map, XContentType.JSON));
|
||||
if (request.numberOfActions() >= 100) {
|
||||
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
// 处理错误
|
||||
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||
} else {
|
||||
processedCount += request.numberOfActions();
|
||||
println "Processed " + processedCount + " documents "+host
|
||||
}
|
||||
request = new BulkRequest();
|
||||
}
|
||||
}
|
||||
|
||||
if (request.numberOfActions() > 0) {
|
||||
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
// 处理错误
|
||||
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||
} else {
|
||||
processedCount += request.numberOfActions();
|
||||
println "Processed " + processedCount + " documents "+host
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def delEsData(id, host, indexName){
|
||||
def header = [:]
|
||||
if(host && host == "192.168.0.89"){
|
||||
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
|
||||
}else if(host && host == "192.168.0.52"){
|
||||
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:goojia123456".getBytes("UTF-8"))}".toString()
|
||||
}
|
||||
String seawaterUrl = "http://${host}:9200/${indexName}/_delete_by_query"
|
||||
def body = """
|
||||
{
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{
|
||||
"match": {
|
||||
"id": "${id}"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
HttpRequest request = new HttpRequest()
|
||||
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
|
||||
println(res)
|
||||
}
|
||||
|
||||
def findMongoDbDatas(cat, city){
|
||||
def q = new BasicDBObject("category", cat).append("city", city)
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("_id",1)
|
||||
def dbcursor = SEAWEED_CONN.find(q, keys)
|
||||
def mongoIds = []
|
||||
for(def d : dbcursor){
|
||||
mongoIds << d._id.toString()
|
||||
}
|
||||
return mongoIds
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user