Merge branch 'script_gj' into 'master'

add

See merge request !1070
This commit is contained in:
2025-02-07 11:43:45 +08:00
@@ -0,0 +1,413 @@
package com.fangjia.script.gj.district
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.FiledStrand
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.bson.types.ObjectId
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 根据链家数据修改小区库中字段
* 使用到中间映射表
*/
@BatchOwner(Watcher.GaoJuan)
class UpdateDistrictField extends BaseBatch{
final DBCollection thirdSeaweedMappingColl = MongoUtils.getCollectionByName("fangjia_craw", "third_seaweed_mapping", "192.168.0.235", 27018)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def fields = ["developer","estate_company","household_count","loudong_count","estate_charge","car_site_count"]
def sources = ["链家", "我爱我家", "安居客"]
def statMap = [:]
static main(args) {
runBatch(new UpdateDistrictField(), args)
}
@Override
void process(Object args) {
def sourceMap = [:]
sources.each {source->
def valMap = findThirdMatchDist(source)
println source +" "+valMap.size()
if(valMap){
sourceMap.put(source, valMap)
}
}
def margeResults = megerDatas(sourceMap)
// updateDB(margeResults)
}
def megerDatas(sourceMap){
def set = new HashSet()
sources.each {source->
def valMap = sourceMap.get(source)
set.addAll(valMap.keySet())
}
def lastValues = []
set.each {id->
def dbObj = CommonDatautils.findDistrictById(id)
def datas = []
sources.each {source->
def valMap = sourceMap.get(source)
def sourceEsObj = findCommunities(valMap.get(id))
if(sourceEsObj){
if(source == "链家"){
sourceEsObj.source = "lianjia"
}else if(source == "我爱我家"){
sourceEsObj.source = "wiwj"
}else if(source == "安居客"){
sourceEsObj.source = "anjuke"
}
datas << sourceEsObj
}
}
if(dbObj && datas){
def newValMap = getNewField2(datas, dbObj)
if(newValMap){
newValMap.put("id", id)
lastValues << newValMap
}
}
}
statMap.each {source,fieldMap->
println source +" "+fieldMap.size()
fieldMap.each{f,v->
println f+" "+ v
}
println "----------------------------------------"
}
return lastValues
}
def margeValue(lianjiaMap, wiwjMap, anjukeMap){
def lianjisIds = lianjiaMap.keySet()
def wiwjIds = wiwjMap.keySet()
def anjukeIds = anjukeMap.keySet()
def set = new HashSet()
set.addAll(lianjisIds)
set.addAll(wiwjIds)
set.addAll(anjukeIds)
def lastValues = []
def lianjiaUpdateCount = 0
def wiwjUpdateCount = 0
def anjukeUpdateCount = 0
def lianjiaCountMap = [:]
def wiwjCountMap = [:]
def anjukeCountMap = [:]
set.each {id->
BasicDBObject newValMap = new BasicDBObject()
fields.each {field->
def lianjiaValMap = lianjiaMap.get(id)
def wiwjValMap = wiwjMap.get(id)
def anjukeValMap = anjukeMap.get(id)
if(lianjiaValMap && lianjiaValMap.containsKey(field)){
newValMap.put(field, lianjiaValMap.get(field))
lianjiaUpdateCount++
def count = lianjiaCountMap.get(field)
if(!count){
lianjiaCountMap.put(field, 1)
}else{
lianjiaCountMap.put(field, count + 1)
}
}else if(wiwjValMap && wiwjValMap.containsKey(field)){
newValMap.put(field, wiwjValMap.get(field))
wiwjUpdateCount++
def count = wiwjCountMap.get(field)
if(!count){
wiwjCountMap.put(field, 1)
}else{
wiwjCountMap.put(field, count + 1)
}
}else if(anjukeValMap && anjukeValMap.containsKey(field)){
newValMap.put(field, anjukeValMap.get(field))
anjukeUpdateCount++
def count = anjukeCountMap.get(field)
if(!count){
anjukeCountMap.put(field, 1)
}else{
anjukeCountMap.put(field, count + 1)
}
}
}
newValMap.put("id", id)
lastValues << newValMap
}
lianjiaCountMap.each {k,v->
println "[链家] "+k+" "+v
}
wiwjCountMap.each {k,v->
println "[我爱我家] "+k+" "+v
}
anjukeCountMap.each {k,v->
println "[安居客] "+k+" "+v
}
return lastValues
}
def updateDB(margeResults){
for(def mr : margeResults){
def id = mr.get("id")
mr.remove("id")
mr.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',mr))
}
}
def batchUpdateDist(source){
// def rets = findThirdMatchDist(source)
println "[${source}] 符保条件小区量:"+rets.size()
def updateIndex = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
def allUpdateList = new CopyOnWriteArrayList()
def sourceMap = new ConcurrentHashMap()
for(def i=0;i<rets.size();i++) {
final def ret = rets.get(i)
// fixedThreadPool.execute(
// new Runnable() {
// void run() {
try {
def newValMap = running(ret.url, ret.old_match_id)
if(newValMap){
sourceMap.put(ret.old_match_id, newValMap)
allUpdateList << newValMap
updateIndex.addAndGet(1)
}
}catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
// }
// }
// )
}
latch.await()
println "总修改量:"+updateIndex.get()
def countMap = [:]
for(def map : allUpdateList){
map.each{k,v->
def count = countMap.get(k)
if(!count){
countMap.put(k, 1)
}else{
countMap.put(k, count+1)
}
}
}
countMap.each {k,v->
println k +" 更新量:"+v
}
println "---------------------------------------------------------"
return sourceMap
}
def running(url, old_match_id){
def esObj = findCommunities(url)
def dbObj = CommonDatautils.findDistrictById(old_match_id)
if(dbObj){
def newValMap = getNewField(esObj, dbObj)
return newValMap
}
return null
}
def getNewField(esObj, dbObj){
BasicDBObject newValMap = new BasicDBObject()
esObj.each{field, esVal->
def dbVal = dbObj.get(field)
def newVal = null
if(esVal && field=="developer" && esVal!=dbVal){
newVal = FiledStrand.processCompany(esVal)
}
if(esVal && field=="estate_company" && esVal!=dbVal){
newVal = FiledStrand.processCompany(esVal)
}
if(esVal && field=="household_count" && esVal!=dbVal){
newVal = FiledStrand.processHouseholdCount(esVal, dbObj.estate_type2)
}
if(esVal && field=="loudong_count" && esVal!=dbVal){
newVal = FiledStrand.processLoudongCount(esVal)
}
if(esVal && field=="estate_charge" && esVal!=dbVal){
def numVal = FiledStrand.getNumber(esVal)
if(numVal && numVal.toString().toDouble()!=dbVal){
newVal = FiledStrand.processEstateCharge(dbObj.city, numVal, dbObj.estate_type2)
newVal = newVal.toString()
}
}
if(esVal && field=="car_site_count"){
def numVal = FiledStrand.getNumber(esVal)
if(numVal && numVal.toString().toInteger()!=dbVal){
newVal = FiledStrand.processCarSite(numVal)
}
}
if(newVal){
newValMap.put(field, newVal)
}
}
return newValMap
}
def getNewField2(esDatas, dbObj){
BasicDBObject newValMap = new BasicDBObject()
def lastValueMap = [:]
def haveValueMap = [:]
fields.each{field->
for(def esData : esDatas){
def esVal = esData.get(field)
def dbVal = dbObj.get(field)
def newVal = null
if(haveValueMap.containsKey(field)){
continue
}else{
if(lastValueMap.containsKey(field)){
continue
}
if(esVal){
if(field=="estate_charge" || field=="car_site_count") {
esVal = FiledStrand.getNumber(esVal)
}
if(dbVal && esVal.toString()==dbVal.toString()){
haveValueMap.put(field, esVal)
}else{
if(field=="developer" && esVal!=dbVal){
newVal = FiledStrand.processCompany(esVal)
}
if(field=="estate_company" && esVal!=dbVal){
newVal = FiledStrand.processCompany(esVal)
}
if(field=="household_count" && esVal!=dbVal){
newVal = FiledStrand.processHouseholdCount(esVal, dbObj.estate_type2)
}
if(esVal && field=="loudong_count" && esVal!=dbVal){
newVal = FiledStrand.processLoudongCount(esVal)
}
if(field=="estate_charge" && esVal!=dbVal){
def numVal = FiledStrand.getNumber(esVal)
if(numVal && numVal.toString().toDouble()!=dbVal){
newVal = FiledStrand.processEstateCharge(dbObj.city, numVal, dbObj.estate_type2)
newVal = newVal.toString()
}
}
if(field=="car_site_count"){
def numVal = FiledStrand.getNumber(esVal)
if(numVal && numVal.toString().toInteger()!=dbVal){
newVal = FiledStrand.processCarSite(numVal)
}
}
}
}
if(newVal){
lastValueMap.put(field, newVal)
//以下为统计各网站各字段的更新情况
def fieldStatMap = statMap.get(esData.source)
if(!fieldStatMap){
fieldStatMap = [:]
statMap.put(esData.source, fieldStatMap)
}
def count = fieldStatMap.get(field)
if(!count){
fieldStatMap.put(field, 1)
}else{
fieldStatMap.put(field, count + 1)
}
}
}
}
}
return lastValueMap
}
def findThirdMatchDist(source){
def query = new BasicDBObject("source",source).append("old_match_way", 1)
def keys = new BasicDBObject()
keys.put("url", 1)
keys.put("old_match_id", 1)
keys.put("old_match_city", 1)
keys.put("old_match_district", 1)
keys.put("old_match_name", 1)
def idsMap = [:]
def dbcursor = thirdSeaweedMappingColl.find(query, keys)
for(def db : dbcursor){
if(!db.old_match_id && db.old_match_name){
def dbo = CommonDatautils.findDistrict(db.old_match_city, db.old_match_district, db.old_match_name, "district")
if(dbo){
idsMap.put(dbo._id.toString(), db.url)
}
}else{
idsMap.put(db.old_match_id, db.url)
}
}
return idsMap
}
static def findCommunities(url){
if(!url){
return null
}
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("url", url))
def rets = []
def datas = null
try{
datas = ElasticUtils.executeSimpleQuery("192.168.0.206", "community", queryBuilder);
}catch(Exception e){
findCommunities(url)
}
def data = [:]
if(datas) {
for (int k = 0; k < datas.size(); k++) {
def d = datas[k];
data.put("household_count", d.households)
data.put("loudong_count", d.loudong_cnt)
data.put("developer", d.developer)
data.put("estate_company", d.property_company)
data.put("estate_charge", d.property_fee)
data.put("car_site_count", d.parking_cnt)
}
}
return data;
}
}