diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index d77e63373..94a3445f5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -5137,17 +5137,22 @@ class CommonDatautils { def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT) def townMap = [:] for(def d : dbcursor){ + def distName = null if(d.mzb_district_name){ - def list = townMap.get(d.mzb_district_name) + distName = d.mzb_district_name + }else{ + distName = d.district + } + if(distName){ + def list = townMap.get(distName) if(!list){ list = [] } list << d - townMap.put(d.mzb_district_name, list) + townMap.put(distName, list) } } - - println "[${city}] 乡镇街道 加载完成 "+new Date().format("yyyy-MM-dd HH:mm:ss") +// println "[${city}] 乡镇街道 加载完成 "+new Date().format("yyyy-MM-dd HH:mm:ss") return townMap } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy index a3c119bec..514dd9631 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy @@ -909,14 +909,18 @@ class FiledStrand { } static def getNumber(def input) { - Pattern pattern = Pattern.compile("\\d+(\\.\\d+)?"); - Matcher matcher = pattern.matcher(input); - List list = new ArrayList(); - while (matcher.find()) { - list.add(matcher.group()); - } - if(list.size()>0){ - return list.get(0); + try{ + Pattern pattern = Pattern.compile("\\d+(\\.\\d+)?"); + Matcher matcher = pattern.matcher(input); + List list = new ArrayList(); + while (matcher.find()) { + list.add(matcher.group()); + } + if(list.size()>0){ + return list.get(0); + } + }catch(Exception e){ + println input+" error" } return null; } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/UpdateDistrictField.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/UpdateDistrictField.groovy new file mode 100644 index 000000000..710eeacca --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/UpdateDistrictField.groovy @@ -0,0 +1,413 @@ +package com.fangjia.script.gj.district + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.FiledStrand +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.bson.types.ObjectId +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 根据链家数据修改小区库中字段 + * 使用到中间映射表 + */ +@BatchOwner(Watcher.GaoJuan) +class UpdateDistrictField extends BaseBatch{ + + final DBCollection thirdSeaweedMappingColl = MongoUtils.getCollectionByName("fangjia_craw", "third_seaweed_mapping", "192.168.0.235", 27018) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + + def fields = ["developer","estate_company","household_count","loudong_count","estate_charge","car_site_count"] + + def sources = ["链家", "我爱我家", "安居客"] + + def statMap = [:] + + static main(args) { + runBatch(new UpdateDistrictField(), args) + } + + @Override + void process(Object args) { + def sourceMap = [:] + sources.each {source-> + def valMap = findThirdMatchDist(source) + println source +" "+valMap.size() + if(valMap){ + sourceMap.put(source, valMap) + } + } + def margeResults = megerDatas(sourceMap) +// updateDB(margeResults) + } + + def megerDatas(sourceMap){ + def set = new HashSet() + sources.each {source-> + def valMap = sourceMap.get(source) + set.addAll(valMap.keySet()) + } + def lastValues = [] + set.each {id-> + + def dbObj = CommonDatautils.findDistrictById(id) + + def datas = [] + sources.each {source-> + def valMap = sourceMap.get(source) + def sourceEsObj = findCommunities(valMap.get(id)) + if(sourceEsObj){ + if(source == "链家"){ + sourceEsObj.source = "lianjia" + }else if(source == "我爱我家"){ + sourceEsObj.source = "wiwj" + }else if(source == "安居客"){ + sourceEsObj.source = "anjuke" + } + datas << sourceEsObj + } + } + if(dbObj && datas){ + def newValMap = getNewField2(datas, dbObj) + if(newValMap){ + newValMap.put("id", id) + lastValues << newValMap + } + } + } + + statMap.each {source,fieldMap-> + println source +" "+fieldMap.size() + fieldMap.each{f,v-> + println f+" "+ v + } + println "----------------------------------------" + } + return lastValues + } + + + def margeValue(lianjiaMap, wiwjMap, anjukeMap){ + def lianjisIds = lianjiaMap.keySet() + def wiwjIds = wiwjMap.keySet() + def anjukeIds = anjukeMap.keySet() + def set = new HashSet() + set.addAll(lianjisIds) + set.addAll(wiwjIds) + set.addAll(anjukeIds) + + def lastValues = [] + def lianjiaUpdateCount = 0 + def wiwjUpdateCount = 0 + def anjukeUpdateCount = 0 + def lianjiaCountMap = [:] + def wiwjCountMap = [:] + def anjukeCountMap = [:] + set.each {id-> + BasicDBObject newValMap = new BasicDBObject() + fields.each {field-> + def lianjiaValMap = lianjiaMap.get(id) + def wiwjValMap = wiwjMap.get(id) + def anjukeValMap = anjukeMap.get(id) + if(lianjiaValMap && lianjiaValMap.containsKey(field)){ + newValMap.put(field, lianjiaValMap.get(field)) + lianjiaUpdateCount++ + def count = lianjiaCountMap.get(field) + if(!count){ + lianjiaCountMap.put(field, 1) + }else{ + lianjiaCountMap.put(field, count + 1) + } + }else if(wiwjValMap && wiwjValMap.containsKey(field)){ + newValMap.put(field, wiwjValMap.get(field)) + wiwjUpdateCount++ + def count = wiwjCountMap.get(field) + if(!count){ + wiwjCountMap.put(field, 1) + }else{ + wiwjCountMap.put(field, count + 1) + } + }else if(anjukeValMap && anjukeValMap.containsKey(field)){ + newValMap.put(field, anjukeValMap.get(field)) + anjukeUpdateCount++ + def count = anjukeCountMap.get(field) + if(!count){ + anjukeCountMap.put(field, 1) + }else{ + anjukeCountMap.put(field, count + 1) + } + } + } + newValMap.put("id", id) + lastValues << newValMap + } + + lianjiaCountMap.each {k,v-> + println "[链家] "+k+" "+v + } + + wiwjCountMap.each {k,v-> + println "[我爱我家] "+k+" "+v + } + + anjukeCountMap.each {k,v-> + println "[安居客] "+k+" "+v + } + return lastValues + } + + def updateDB(margeResults){ + for(def mr : margeResults){ + def id = mr.get("id") + mr.remove("id") + mr.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',mr)) + } + } + + def batchUpdateDist(source){ +// def rets = findThirdMatchDist(source) + println "[${source}] 符保条件小区量:"+rets.size() + def updateIndex = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(rets.size()) + def allUpdateList = new CopyOnWriteArrayList() + + def sourceMap = new ConcurrentHashMap() + + for(def i=0;i + def count = countMap.get(k) + if(!count){ + countMap.put(k, 1) + }else{ + countMap.put(k, count+1) + } + } + } + + countMap.each {k,v-> + println k +" 更新量:"+v + } + + println "---------------------------------------------------------" + return sourceMap + } + + def running(url, old_match_id){ + def esObj = findCommunities(url) + def dbObj = CommonDatautils.findDistrictById(old_match_id) + if(dbObj){ + def newValMap = getNewField(esObj, dbObj) + return newValMap + } + return null + } + + def getNewField(esObj, dbObj){ + BasicDBObject newValMap = new BasicDBObject() + esObj.each{field, esVal-> + def dbVal = dbObj.get(field) + def newVal = null + if(esVal && field=="developer" && esVal!=dbVal){ + newVal = FiledStrand.processCompany(esVal) + + } + if(esVal && field=="estate_company" && esVal!=dbVal){ + newVal = FiledStrand.processCompany(esVal) + } + + if(esVal && field=="household_count" && esVal!=dbVal){ + newVal = FiledStrand.processHouseholdCount(esVal, dbObj.estate_type2) + } + + if(esVal && field=="loudong_count" && esVal!=dbVal){ + newVal = FiledStrand.processLoudongCount(esVal) + } + + if(esVal && field=="estate_charge" && esVal!=dbVal){ + def numVal = FiledStrand.getNumber(esVal) + if(numVal && numVal.toString().toDouble()!=dbVal){ + newVal = FiledStrand.processEstateCharge(dbObj.city, numVal, dbObj.estate_type2) + newVal = newVal.toString() + } + } + if(esVal && field=="car_site_count"){ + def numVal = FiledStrand.getNumber(esVal) + if(numVal && numVal.toString().toInteger()!=dbVal){ + newVal = FiledStrand.processCarSite(numVal) + } + } + if(newVal){ + newValMap.put(field, newVal) + } + + } + + return newValMap + } + + def getNewField2(esDatas, dbObj){ + BasicDBObject newValMap = new BasicDBObject() + def lastValueMap = [:] + def haveValueMap = [:] + fields.each{field-> + for(def esData : esDatas){ + def esVal = esData.get(field) + def dbVal = dbObj.get(field) + def newVal = null + if(haveValueMap.containsKey(field)){ + continue + }else{ + if(lastValueMap.containsKey(field)){ + continue + } + if(esVal){ + + if(field=="estate_charge" || field=="car_site_count") { + esVal = FiledStrand.getNumber(esVal) + } + if(dbVal && esVal.toString()==dbVal.toString()){ + haveValueMap.put(field, esVal) + }else{ + if(field=="developer" && esVal!=dbVal){ + newVal = FiledStrand.processCompany(esVal) + } + + if(field=="estate_company" && esVal!=dbVal){ + newVal = FiledStrand.processCompany(esVal) + } + + if(field=="household_count" && esVal!=dbVal){ + newVal = FiledStrand.processHouseholdCount(esVal, dbObj.estate_type2) + } + + if(esVal && field=="loudong_count" && esVal!=dbVal){ + newVal = FiledStrand.processLoudongCount(esVal) + } + + if(field=="estate_charge" && esVal!=dbVal){ + def numVal = FiledStrand.getNumber(esVal) + if(numVal && numVal.toString().toDouble()!=dbVal){ + newVal = FiledStrand.processEstateCharge(dbObj.city, numVal, dbObj.estate_type2) + newVal = newVal.toString() + } + } + if(field=="car_site_count"){ + def numVal = FiledStrand.getNumber(esVal) + if(numVal && numVal.toString().toInteger()!=dbVal){ + newVal = FiledStrand.processCarSite(numVal) + } + } + } + } + if(newVal){ + lastValueMap.put(field, newVal) + + //以下为统计各网站各字段的更新情况 + def fieldStatMap = statMap.get(esData.source) + if(!fieldStatMap){ + fieldStatMap = [:] + statMap.put(esData.source, fieldStatMap) + } + def count = fieldStatMap.get(field) + if(!count){ + fieldStatMap.put(field, 1) + }else{ + fieldStatMap.put(field, count + 1) + } + } + } + } + } + return lastValueMap + } + + def findThirdMatchDist(source){ + def query = new BasicDBObject("source",source).append("old_match_way", 1) + def keys = new BasicDBObject() + keys.put("url", 1) + keys.put("old_match_id", 1) + keys.put("old_match_city", 1) + keys.put("old_match_district", 1) + keys.put("old_match_name", 1) + def idsMap = [:] + def dbcursor = thirdSeaweedMappingColl.find(query, keys) + for(def db : dbcursor){ + if(!db.old_match_id && db.old_match_name){ + def dbo = CommonDatautils.findDistrict(db.old_match_city, db.old_match_district, db.old_match_name, "district") + if(dbo){ + idsMap.put(dbo._id.toString(), db.url) + } + }else{ + idsMap.put(db.old_match_id, db.url) + } + + } + return idsMap + } + + static def findCommunities(url){ + if(!url){ + return null + } + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("url", url)) + def rets = [] + def datas = null + try{ + datas = ElasticUtils.executeSimpleQuery("192.168.0.206", "community", queryBuilder); + }catch(Exception e){ + findCommunities(url) + } + def data = [:] + if(datas) { + for (int k = 0; k < datas.size(); k++) { + def d = datas[k]; + data.put("household_count", d.households) + data.put("loudong_count", d.loudong_cnt) + data.put("developer", d.developer) + data.put("estate_company", d.property_company) + data.put("estate_charge", d.property_fee) + data.put("car_site_count", d.parking_cnt) + } + } + return data; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy index 9be75aeb0..e07ac0739 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy @@ -26,7 +26,8 @@ class EsMonthHouseStat { // task.statHouseUpdateCount() // task.statDistHouse() - task.statRent() +// task.statRent() + task.statOfficeHouse() } def statRent(){ @@ -201,6 +202,51 @@ class EsMonthHouseStat { },0) } + def statOfficeHouse(){ + def citys = CommonDatautils.findAllCity() + def allLists = [] + allLists << ["城市", "近一年写字楼出售量", "写字楼量", "近一年出售房源量大于3的写字楼量", "大于3写字楼量占比"] + citys.each{c-> + def count = findHouseCountByCat(c, "office_sell",20240101, 20250201) + + def districts = CommonDatautils.findAllDistricts(c, "office", ["city", "region", "name", "alias"]); + def disthouseCount = 0 + for(def dist : districts){ + def alias = [] + if(dist.alias){ + if(!dist.alias.contains(dist.name)){ + dist.alias << dist.name + } + alias = dist.alias + }else{ + alias << dist.name + } + + def distCount = 0 + if(count){ + distCount = findHouseCountByCat(dist.city, dist.region, alias, 20240101, 20250201, "office_sell") + if(distCount && distCount>=3){ + disthouseCount++ + } + } +// println dist.city+" "+dist.region+" "+dist.name+" "+distCount + } + + println "disthouseCount:"+disthouseCount +// println districts.size() + def precent = 0 + if(districts){ + precent = DataHandleUtils.getDoubleRound(2, disthouseCount*100.0/districts.size()) + } + def row = [c , count, districts.size(), disthouseCount, precent+"%"] + println row + allLists << row + } + + ExcelUtils.write("D:\\数据\\写字楼\\全国写字楼情况占比统计.xlsx", allLists, "写字楼情况") + + } + def findHouseCount(def city, def region, def alias, def startDate, def endDate){ def aliasStr = null; @@ -219,6 +265,23 @@ class EsMonthHouseStat { return total } + def findHouseCountByCat(def city, def region, def alias, def startDate, def endDate, cat){ + def aliasStr = null; + def total = 0 + try{ + aliasStr = escapeExprSpecialWord(alias.join("','")) + String sql = "select count(*) as c from latest_seawater where cat='${cat}' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}') " + + ElasticUtils.executeSqlQuery(sql,{data-> + total = data.c + }) + }catch(Exception e){ + e.printStackTrace() + } + + return total + } + /** * 转义字符串中的特殊字符 * 仅过滤Lucene regexps, 其他的勿用 @@ -240,6 +303,17 @@ class EsMonthHouseStat { } + def findHouseCountByCat(def city, def cat, def startDate, def endDate){ + + String sql = "select count(*) as c from latest_seawater where cat='${cat}' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} " + def total = 0 + ElasticUtils.executeSqlQuery(sql,{ data-> + total = data.c + }) + return total + } + +