+413
@@ -0,0 +1,413 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.FiledStrand
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/**
|
||||
* 根据链家数据修改小区库中字段
|
||||
* 使用到中间映射表
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class UpdateDistrictField extends BaseBatch{
|
||||
|
||||
final DBCollection thirdSeaweedMappingColl = MongoUtils.getCollectionByName("fangjia_craw", "third_seaweed_mapping", "192.168.0.235", 27018)
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
def fields = ["developer","estate_company","household_count","loudong_count","estate_charge","car_site_count"]
|
||||
|
||||
def sources = ["链家", "我爱我家", "安居客"]
|
||||
|
||||
def statMap = [:]
|
||||
|
||||
static main(args) {
|
||||
runBatch(new UpdateDistrictField(), args)
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
def sourceMap = [:]
|
||||
sources.each {source->
|
||||
def valMap = findThirdMatchDist(source)
|
||||
println source +" "+valMap.size()
|
||||
if(valMap){
|
||||
sourceMap.put(source, valMap)
|
||||
}
|
||||
}
|
||||
def margeResults = megerDatas(sourceMap)
|
||||
// updateDB(margeResults)
|
||||
}
|
||||
|
||||
def megerDatas(sourceMap){
|
||||
def set = new HashSet()
|
||||
sources.each {source->
|
||||
def valMap = sourceMap.get(source)
|
||||
set.addAll(valMap.keySet())
|
||||
}
|
||||
def lastValues = []
|
||||
set.each {id->
|
||||
|
||||
def dbObj = CommonDatautils.findDistrictById(id)
|
||||
|
||||
def datas = []
|
||||
sources.each {source->
|
||||
def valMap = sourceMap.get(source)
|
||||
def sourceEsObj = findCommunities(valMap.get(id))
|
||||
if(sourceEsObj){
|
||||
if(source == "链家"){
|
||||
sourceEsObj.source = "lianjia"
|
||||
}else if(source == "我爱我家"){
|
||||
sourceEsObj.source = "wiwj"
|
||||
}else if(source == "安居客"){
|
||||
sourceEsObj.source = "anjuke"
|
||||
}
|
||||
datas << sourceEsObj
|
||||
}
|
||||
}
|
||||
if(dbObj && datas){
|
||||
def newValMap = getNewField2(datas, dbObj)
|
||||
if(newValMap){
|
||||
newValMap.put("id", id)
|
||||
lastValues << newValMap
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
statMap.each {source,fieldMap->
|
||||
println source +" "+fieldMap.size()
|
||||
fieldMap.each{f,v->
|
||||
println f+" "+ v
|
||||
}
|
||||
println "----------------------------------------"
|
||||
}
|
||||
return lastValues
|
||||
}
|
||||
|
||||
|
||||
def margeValue(lianjiaMap, wiwjMap, anjukeMap){
|
||||
def lianjisIds = lianjiaMap.keySet()
|
||||
def wiwjIds = wiwjMap.keySet()
|
||||
def anjukeIds = anjukeMap.keySet()
|
||||
def set = new HashSet()
|
||||
set.addAll(lianjisIds)
|
||||
set.addAll(wiwjIds)
|
||||
set.addAll(anjukeIds)
|
||||
|
||||
def lastValues = []
|
||||
def lianjiaUpdateCount = 0
|
||||
def wiwjUpdateCount = 0
|
||||
def anjukeUpdateCount = 0
|
||||
def lianjiaCountMap = [:]
|
||||
def wiwjCountMap = [:]
|
||||
def anjukeCountMap = [:]
|
||||
set.each {id->
|
||||
BasicDBObject newValMap = new BasicDBObject()
|
||||
fields.each {field->
|
||||
def lianjiaValMap = lianjiaMap.get(id)
|
||||
def wiwjValMap = wiwjMap.get(id)
|
||||
def anjukeValMap = anjukeMap.get(id)
|
||||
if(lianjiaValMap && lianjiaValMap.containsKey(field)){
|
||||
newValMap.put(field, lianjiaValMap.get(field))
|
||||
lianjiaUpdateCount++
|
||||
def count = lianjiaCountMap.get(field)
|
||||
if(!count){
|
||||
lianjiaCountMap.put(field, 1)
|
||||
}else{
|
||||
lianjiaCountMap.put(field, count + 1)
|
||||
}
|
||||
}else if(wiwjValMap && wiwjValMap.containsKey(field)){
|
||||
newValMap.put(field, wiwjValMap.get(field))
|
||||
wiwjUpdateCount++
|
||||
def count = wiwjCountMap.get(field)
|
||||
if(!count){
|
||||
wiwjCountMap.put(field, 1)
|
||||
}else{
|
||||
wiwjCountMap.put(field, count + 1)
|
||||
}
|
||||
}else if(anjukeValMap && anjukeValMap.containsKey(field)){
|
||||
newValMap.put(field, anjukeValMap.get(field))
|
||||
anjukeUpdateCount++
|
||||
def count = anjukeCountMap.get(field)
|
||||
if(!count){
|
||||
anjukeCountMap.put(field, 1)
|
||||
}else{
|
||||
anjukeCountMap.put(field, count + 1)
|
||||
}
|
||||
}
|
||||
}
|
||||
newValMap.put("id", id)
|
||||
lastValues << newValMap
|
||||
}
|
||||
|
||||
lianjiaCountMap.each {k,v->
|
||||
println "[链家] "+k+" "+v
|
||||
}
|
||||
|
||||
wiwjCountMap.each {k,v->
|
||||
println "[我爱我家] "+k+" "+v
|
||||
}
|
||||
|
||||
anjukeCountMap.each {k,v->
|
||||
println "[安居客] "+k+" "+v
|
||||
}
|
||||
return lastValues
|
||||
}
|
||||
|
||||
def updateDB(margeResults){
|
||||
for(def mr : margeResults){
|
||||
def id = mr.get("id")
|
||||
mr.remove("id")
|
||||
mr.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',mr))
|
||||
}
|
||||
}
|
||||
|
||||
def batchUpdateDist(source){
|
||||
// def rets = findThirdMatchDist(source)
|
||||
println "[${source}] 符保条件小区量:"+rets.size()
|
||||
def updateIndex = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(rets.size())
|
||||
def allUpdateList = new CopyOnWriteArrayList()
|
||||
|
||||
def sourceMap = new ConcurrentHashMap()
|
||||
|
||||
for(def i=0;i<rets.size();i++) {
|
||||
final def ret = rets.get(i)
|
||||
// fixedThreadPool.execute(
|
||||
// new Runnable() {
|
||||
// void run() {
|
||||
try {
|
||||
def newValMap = running(ret.url, ret.old_match_id)
|
||||
if(newValMap){
|
||||
sourceMap.put(ret.old_match_id, newValMap)
|
||||
allUpdateList << newValMap
|
||||
updateIndex.addAndGet(1)
|
||||
}
|
||||
}catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
// }
|
||||
// }
|
||||
// )
|
||||
}
|
||||
latch.await()
|
||||
println "总修改量:"+updateIndex.get()
|
||||
def countMap = [:]
|
||||
for(def map : allUpdateList){
|
||||
map.each{k,v->
|
||||
def count = countMap.get(k)
|
||||
if(!count){
|
||||
countMap.put(k, 1)
|
||||
}else{
|
||||
countMap.put(k, count+1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
countMap.each {k,v->
|
||||
println k +" 更新量:"+v
|
||||
}
|
||||
|
||||
println "---------------------------------------------------------"
|
||||
return sourceMap
|
||||
}
|
||||
|
||||
def running(url, old_match_id){
|
||||
def esObj = findCommunities(url)
|
||||
def dbObj = CommonDatautils.findDistrictById(old_match_id)
|
||||
if(dbObj){
|
||||
def newValMap = getNewField(esObj, dbObj)
|
||||
return newValMap
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def getNewField(esObj, dbObj){
|
||||
BasicDBObject newValMap = new BasicDBObject()
|
||||
esObj.each{field, esVal->
|
||||
def dbVal = dbObj.get(field)
|
||||
def newVal = null
|
||||
if(esVal && field=="developer" && esVal!=dbVal){
|
||||
newVal = FiledStrand.processCompany(esVal)
|
||||
|
||||
}
|
||||
if(esVal && field=="estate_company" && esVal!=dbVal){
|
||||
newVal = FiledStrand.processCompany(esVal)
|
||||
}
|
||||
|
||||
if(esVal && field=="household_count" && esVal!=dbVal){
|
||||
newVal = FiledStrand.processHouseholdCount(esVal, dbObj.estate_type2)
|
||||
}
|
||||
|
||||
if(esVal && field=="loudong_count" && esVal!=dbVal){
|
||||
newVal = FiledStrand.processLoudongCount(esVal)
|
||||
}
|
||||
|
||||
if(esVal && field=="estate_charge" && esVal!=dbVal){
|
||||
def numVal = FiledStrand.getNumber(esVal)
|
||||
if(numVal && numVal.toString().toDouble()!=dbVal){
|
||||
newVal = FiledStrand.processEstateCharge(dbObj.city, numVal, dbObj.estate_type2)
|
||||
newVal = newVal.toString()
|
||||
}
|
||||
}
|
||||
if(esVal && field=="car_site_count"){
|
||||
def numVal = FiledStrand.getNumber(esVal)
|
||||
if(numVal && numVal.toString().toInteger()!=dbVal){
|
||||
newVal = FiledStrand.processCarSite(numVal)
|
||||
}
|
||||
}
|
||||
if(newVal){
|
||||
newValMap.put(field, newVal)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return newValMap
|
||||
}
|
||||
|
||||
def getNewField2(esDatas, dbObj){
|
||||
BasicDBObject newValMap = new BasicDBObject()
|
||||
def lastValueMap = [:]
|
||||
def haveValueMap = [:]
|
||||
fields.each{field->
|
||||
for(def esData : esDatas){
|
||||
def esVal = esData.get(field)
|
||||
def dbVal = dbObj.get(field)
|
||||
def newVal = null
|
||||
if(haveValueMap.containsKey(field)){
|
||||
continue
|
||||
}else{
|
||||
if(lastValueMap.containsKey(field)){
|
||||
continue
|
||||
}
|
||||
if(esVal){
|
||||
|
||||
if(field=="estate_charge" || field=="car_site_count") {
|
||||
esVal = FiledStrand.getNumber(esVal)
|
||||
}
|
||||
if(dbVal && esVal.toString()==dbVal.toString()){
|
||||
haveValueMap.put(field, esVal)
|
||||
}else{
|
||||
if(field=="developer" && esVal!=dbVal){
|
||||
newVal = FiledStrand.processCompany(esVal)
|
||||
}
|
||||
|
||||
if(field=="estate_company" && esVal!=dbVal){
|
||||
newVal = FiledStrand.processCompany(esVal)
|
||||
}
|
||||
|
||||
if(field=="household_count" && esVal!=dbVal){
|
||||
newVal = FiledStrand.processHouseholdCount(esVal, dbObj.estate_type2)
|
||||
}
|
||||
|
||||
if(esVal && field=="loudong_count" && esVal!=dbVal){
|
||||
newVal = FiledStrand.processLoudongCount(esVal)
|
||||
}
|
||||
|
||||
if(field=="estate_charge" && esVal!=dbVal){
|
||||
def numVal = FiledStrand.getNumber(esVal)
|
||||
if(numVal && numVal.toString().toDouble()!=dbVal){
|
||||
newVal = FiledStrand.processEstateCharge(dbObj.city, numVal, dbObj.estate_type2)
|
||||
newVal = newVal.toString()
|
||||
}
|
||||
}
|
||||
if(field=="car_site_count"){
|
||||
def numVal = FiledStrand.getNumber(esVal)
|
||||
if(numVal && numVal.toString().toInteger()!=dbVal){
|
||||
newVal = FiledStrand.processCarSite(numVal)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if(newVal){
|
||||
lastValueMap.put(field, newVal)
|
||||
|
||||
//以下为统计各网站各字段的更新情况
|
||||
def fieldStatMap = statMap.get(esData.source)
|
||||
if(!fieldStatMap){
|
||||
fieldStatMap = [:]
|
||||
statMap.put(esData.source, fieldStatMap)
|
||||
}
|
||||
def count = fieldStatMap.get(field)
|
||||
if(!count){
|
||||
fieldStatMap.put(field, 1)
|
||||
}else{
|
||||
fieldStatMap.put(field, count + 1)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return lastValueMap
|
||||
}
|
||||
|
||||
def findThirdMatchDist(source){
|
||||
def query = new BasicDBObject("source",source).append("old_match_way", 1)
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("url", 1)
|
||||
keys.put("old_match_id", 1)
|
||||
keys.put("old_match_city", 1)
|
||||
keys.put("old_match_district", 1)
|
||||
keys.put("old_match_name", 1)
|
||||
def idsMap = [:]
|
||||
def dbcursor = thirdSeaweedMappingColl.find(query, keys)
|
||||
for(def db : dbcursor){
|
||||
if(!db.old_match_id && db.old_match_name){
|
||||
def dbo = CommonDatautils.findDistrict(db.old_match_city, db.old_match_district, db.old_match_name, "district")
|
||||
if(dbo){
|
||||
idsMap.put(dbo._id.toString(), db.url)
|
||||
}
|
||||
}else{
|
||||
idsMap.put(db.old_match_id, db.url)
|
||||
}
|
||||
|
||||
}
|
||||
return idsMap
|
||||
}
|
||||
|
||||
static def findCommunities(url){
|
||||
if(!url){
|
||||
return null
|
||||
}
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("url", url))
|
||||
def rets = []
|
||||
def datas = null
|
||||
try{
|
||||
datas = ElasticUtils.executeSimpleQuery("192.168.0.206", "community", queryBuilder);
|
||||
}catch(Exception e){
|
||||
findCommunities(url)
|
||||
}
|
||||
def data = [:]
|
||||
if(datas) {
|
||||
for (int k = 0; k < datas.size(); k++) {
|
||||
def d = datas[k];
|
||||
data.put("household_count", d.households)
|
||||
data.put("loudong_count", d.loudong_cnt)
|
||||
data.put("developer", d.developer)
|
||||
data.put("estate_company", d.property_company)
|
||||
data.put("estate_charge", d.property_fee)
|
||||
data.put("car_site_count", d.parking_cnt)
|
||||
}
|
||||
}
|
||||
return data;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user