From 02c5f23fbf9ebc0c713f4d46928c8ac89e9cc5ca Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Fri, 7 Feb 2025 11:35:31 +0800 Subject: [PATCH] add --- .../gj/district/UpdateDistrictField.groovy | 413 ++++++++++++++++++ 1 file changed, 413 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/UpdateDistrictField.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/UpdateDistrictField.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/UpdateDistrictField.groovy new file mode 100644 index 000000000..710eeacca --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/UpdateDistrictField.groovy @@ -0,0 +1,413 @@ +package com.fangjia.script.gj.district + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.FiledStrand +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.bson.types.ObjectId +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 根据链家数据修改小区库中字段 + * 使用到中间映射表 + */ +@BatchOwner(Watcher.GaoJuan) +class UpdateDistrictField extends BaseBatch{ + + final DBCollection thirdSeaweedMappingColl = MongoUtils.getCollectionByName("fangjia_craw", "third_seaweed_mapping", "192.168.0.235", 27018) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + + def fields = ["developer","estate_company","household_count","loudong_count","estate_charge","car_site_count"] + + def sources = ["链家", "我爱我家", "安居客"] + + def statMap = [:] + + static main(args) { + runBatch(new UpdateDistrictField(), args) + } + + @Override + void process(Object args) { + def sourceMap = [:] + sources.each {source-> + def valMap = findThirdMatchDist(source) + println source +" "+valMap.size() + if(valMap){ + sourceMap.put(source, valMap) + } + } + def margeResults = megerDatas(sourceMap) +// updateDB(margeResults) + } + + def megerDatas(sourceMap){ + def set = new HashSet() + sources.each {source-> + def valMap = sourceMap.get(source) + set.addAll(valMap.keySet()) + } + def lastValues = [] + set.each {id-> + + def dbObj = CommonDatautils.findDistrictById(id) + + def datas = [] + sources.each {source-> + def valMap = sourceMap.get(source) + def sourceEsObj = findCommunities(valMap.get(id)) + if(sourceEsObj){ + if(source == "链家"){ + sourceEsObj.source = "lianjia" + }else if(source == "我爱我家"){ + sourceEsObj.source = "wiwj" + }else if(source == "安居客"){ + sourceEsObj.source = "anjuke" + } + datas << sourceEsObj + } + } + if(dbObj && datas){ + def newValMap = getNewField2(datas, dbObj) + if(newValMap){ + newValMap.put("id", id) + lastValues << newValMap + } + } + } + + statMap.each {source,fieldMap-> + println source +" "+fieldMap.size() + fieldMap.each{f,v-> + println f+" "+ v + } + println "----------------------------------------" + } + return lastValues + } + + + def margeValue(lianjiaMap, wiwjMap, anjukeMap){ + def lianjisIds = lianjiaMap.keySet() + def wiwjIds = wiwjMap.keySet() + def anjukeIds = anjukeMap.keySet() + def set = new HashSet() + set.addAll(lianjisIds) + set.addAll(wiwjIds) + set.addAll(anjukeIds) + + def lastValues = [] + def lianjiaUpdateCount = 0 + def wiwjUpdateCount = 0 + def anjukeUpdateCount = 0 + def lianjiaCountMap = [:] + def wiwjCountMap = [:] + def anjukeCountMap = [:] + set.each {id-> + BasicDBObject newValMap = new BasicDBObject() + fields.each {field-> + def lianjiaValMap = lianjiaMap.get(id) + def wiwjValMap = wiwjMap.get(id) + def anjukeValMap = anjukeMap.get(id) + if(lianjiaValMap && lianjiaValMap.containsKey(field)){ + newValMap.put(field, lianjiaValMap.get(field)) + lianjiaUpdateCount++ + def count = lianjiaCountMap.get(field) + if(!count){ + lianjiaCountMap.put(field, 1) + }else{ + lianjiaCountMap.put(field, count + 1) + } + }else if(wiwjValMap && wiwjValMap.containsKey(field)){ + newValMap.put(field, wiwjValMap.get(field)) + wiwjUpdateCount++ + def count = wiwjCountMap.get(field) + if(!count){ + wiwjCountMap.put(field, 1) + }else{ + wiwjCountMap.put(field, count + 1) + } + }else if(anjukeValMap && anjukeValMap.containsKey(field)){ + newValMap.put(field, anjukeValMap.get(field)) + anjukeUpdateCount++ + def count = anjukeCountMap.get(field) + if(!count){ + anjukeCountMap.put(field, 1) + }else{ + anjukeCountMap.put(field, count + 1) + } + } + } + newValMap.put("id", id) + lastValues << newValMap + } + + lianjiaCountMap.each {k,v-> + println "[链家] "+k+" "+v + } + + wiwjCountMap.each {k,v-> + println "[我爱我家] "+k+" "+v + } + + anjukeCountMap.each {k,v-> + println "[安居客] "+k+" "+v + } + return lastValues + } + + def updateDB(margeResults){ + for(def mr : margeResults){ + def id = mr.get("id") + mr.remove("id") + mr.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',mr)) + } + } + + def batchUpdateDist(source){ +// def rets = findThirdMatchDist(source) + println "[${source}] 符保条件小区量:"+rets.size() + def updateIndex = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(rets.size()) + def allUpdateList = new CopyOnWriteArrayList() + + def sourceMap = new ConcurrentHashMap() + + for(def i=0;i + def count = countMap.get(k) + if(!count){ + countMap.put(k, 1) + }else{ + countMap.put(k, count+1) + } + } + } + + countMap.each {k,v-> + println k +" 更新量:"+v + } + + println "---------------------------------------------------------" + return sourceMap + } + + def running(url, old_match_id){ + def esObj = findCommunities(url) + def dbObj = CommonDatautils.findDistrictById(old_match_id) + if(dbObj){ + def newValMap = getNewField(esObj, dbObj) + return newValMap + } + return null + } + + def getNewField(esObj, dbObj){ + BasicDBObject newValMap = new BasicDBObject() + esObj.each{field, esVal-> + def dbVal = dbObj.get(field) + def newVal = null + if(esVal && field=="developer" && esVal!=dbVal){ + newVal = FiledStrand.processCompany(esVal) + + } + if(esVal && field=="estate_company" && esVal!=dbVal){ + newVal = FiledStrand.processCompany(esVal) + } + + if(esVal && field=="household_count" && esVal!=dbVal){ + newVal = FiledStrand.processHouseholdCount(esVal, dbObj.estate_type2) + } + + if(esVal && field=="loudong_count" && esVal!=dbVal){ + newVal = FiledStrand.processLoudongCount(esVal) + } + + if(esVal && field=="estate_charge" && esVal!=dbVal){ + def numVal = FiledStrand.getNumber(esVal) + if(numVal && numVal.toString().toDouble()!=dbVal){ + newVal = FiledStrand.processEstateCharge(dbObj.city, numVal, dbObj.estate_type2) + newVal = newVal.toString() + } + } + if(esVal && field=="car_site_count"){ + def numVal = FiledStrand.getNumber(esVal) + if(numVal && numVal.toString().toInteger()!=dbVal){ + newVal = FiledStrand.processCarSite(numVal) + } + } + if(newVal){ + newValMap.put(field, newVal) + } + + } + + return newValMap + } + + def getNewField2(esDatas, dbObj){ + BasicDBObject newValMap = new BasicDBObject() + def lastValueMap = [:] + def haveValueMap = [:] + fields.each{field-> + for(def esData : esDatas){ + def esVal = esData.get(field) + def dbVal = dbObj.get(field) + def newVal = null + if(haveValueMap.containsKey(field)){ + continue + }else{ + if(lastValueMap.containsKey(field)){ + continue + } + if(esVal){ + + if(field=="estate_charge" || field=="car_site_count") { + esVal = FiledStrand.getNumber(esVal) + } + if(dbVal && esVal.toString()==dbVal.toString()){ + haveValueMap.put(field, esVal) + }else{ + if(field=="developer" && esVal!=dbVal){ + newVal = FiledStrand.processCompany(esVal) + } + + if(field=="estate_company" && esVal!=dbVal){ + newVal = FiledStrand.processCompany(esVal) + } + + if(field=="household_count" && esVal!=dbVal){ + newVal = FiledStrand.processHouseholdCount(esVal, dbObj.estate_type2) + } + + if(esVal && field=="loudong_count" && esVal!=dbVal){ + newVal = FiledStrand.processLoudongCount(esVal) + } + + if(field=="estate_charge" && esVal!=dbVal){ + def numVal = FiledStrand.getNumber(esVal) + if(numVal && numVal.toString().toDouble()!=dbVal){ + newVal = FiledStrand.processEstateCharge(dbObj.city, numVal, dbObj.estate_type2) + newVal = newVal.toString() + } + } + if(field=="car_site_count"){ + def numVal = FiledStrand.getNumber(esVal) + if(numVal && numVal.toString().toInteger()!=dbVal){ + newVal = FiledStrand.processCarSite(numVal) + } + } + } + } + if(newVal){ + lastValueMap.put(field, newVal) + + //以下为统计各网站各字段的更新情况 + def fieldStatMap = statMap.get(esData.source) + if(!fieldStatMap){ + fieldStatMap = [:] + statMap.put(esData.source, fieldStatMap) + } + def count = fieldStatMap.get(field) + if(!count){ + fieldStatMap.put(field, 1) + }else{ + fieldStatMap.put(field, count + 1) + } + } + } + } + } + return lastValueMap + } + + def findThirdMatchDist(source){ + def query = new BasicDBObject("source",source).append("old_match_way", 1) + def keys = new BasicDBObject() + keys.put("url", 1) + keys.put("old_match_id", 1) + keys.put("old_match_city", 1) + keys.put("old_match_district", 1) + keys.put("old_match_name", 1) + def idsMap = [:] + def dbcursor = thirdSeaweedMappingColl.find(query, keys) + for(def db : dbcursor){ + if(!db.old_match_id && db.old_match_name){ + def dbo = CommonDatautils.findDistrict(db.old_match_city, db.old_match_district, db.old_match_name, "district") + if(dbo){ + idsMap.put(dbo._id.toString(), db.url) + } + }else{ + idsMap.put(db.old_match_id, db.url) + } + + } + return idsMap + } + + static def findCommunities(url){ + if(!url){ + return null + } + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("url", url)) + def rets = [] + def datas = null + try{ + datas = ElasticUtils.executeSimpleQuery("192.168.0.206", "community", queryBuilder); + }catch(Exception e){ + findCommunities(url) + } + def data = [:] + if(datas) { + for (int k = 0; k < datas.size(); k++) { + def d = datas[k]; + data.put("household_count", d.households) + data.put("loudong_count", d.loudong_cnt) + data.put("developer", d.developer) + data.put("estate_company", d.property_company) + data.put("estate_charge", d.property_fee) + data.put("car_site_count", d.parking_cnt) + } + } + return data; + } +}