Merge remote-tracking branch 'origin/master'

This commit is contained in:
majintao
2025-02-07 14:10:31 +08:00
4 changed files with 509 additions and 13 deletions
@@ -5137,17 +5137,22 @@ class CommonDatautils {
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def townMap = [:]
for(def d : dbcursor){
def distName = null
if(d.mzb_district_name){
def list = townMap.get(d.mzb_district_name)
distName = d.mzb_district_name
}else{
distName = d.district
}
if(distName){
def list = townMap.get(distName)
if(!list){
list = []
}
list << d
townMap.put(d.mzb_district_name, list)
townMap.put(distName, list)
}
}
println "[${city}] 乡镇街道 加载完成 "+new Date().format("yyyy-MM-dd HH:mm:ss")
// println "[${city}] 乡镇街道 加载完成 "+new Date().format("yyyy-MM-dd HH:mm:ss")
return townMap
}
@@ -909,14 +909,18 @@ class FiledStrand {
}
static def getNumber(def input) {
Pattern pattern = Pattern.compile("\\d+(\\.\\d+)?");
Matcher matcher = pattern.matcher(input);
List<String> list = new ArrayList();
while (matcher.find()) {
list.add(matcher.group());
}
if(list.size()>0){
return list.get(0);
try{
Pattern pattern = Pattern.compile("\\d+(\\.\\d+)?");
Matcher matcher = pattern.matcher(input);
List<String> list = new ArrayList();
while (matcher.find()) {
list.add(matcher.group());
}
if(list.size()>0){
return list.get(0);
}
}catch(Exception e){
println input+" error"
}
return null;
}
@@ -0,0 +1,413 @@
package com.fangjia.script.gj.district
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.FiledStrand
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.bson.types.ObjectId
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 根据链家数据修改小区库中字段
* 使用到中间映射表
*/
@BatchOwner(Watcher.GaoJuan)
class UpdateDistrictField extends BaseBatch{
final DBCollection thirdSeaweedMappingColl = MongoUtils.getCollectionByName("fangjia_craw", "third_seaweed_mapping", "192.168.0.235", 27018)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def fields = ["developer","estate_company","household_count","loudong_count","estate_charge","car_site_count"]
def sources = ["链家", "我爱我家", "安居客"]
def statMap = [:]
static main(args) {
runBatch(new UpdateDistrictField(), args)
}
@Override
void process(Object args) {
def sourceMap = [:]
sources.each {source->
def valMap = findThirdMatchDist(source)
println source +" "+valMap.size()
if(valMap){
sourceMap.put(source, valMap)
}
}
def margeResults = megerDatas(sourceMap)
// updateDB(margeResults)
}
def megerDatas(sourceMap){
def set = new HashSet()
sources.each {source->
def valMap = sourceMap.get(source)
set.addAll(valMap.keySet())
}
def lastValues = []
set.each {id->
def dbObj = CommonDatautils.findDistrictById(id)
def datas = []
sources.each {source->
def valMap = sourceMap.get(source)
def sourceEsObj = findCommunities(valMap.get(id))
if(sourceEsObj){
if(source == "链家"){
sourceEsObj.source = "lianjia"
}else if(source == "我爱我家"){
sourceEsObj.source = "wiwj"
}else if(source == "安居客"){
sourceEsObj.source = "anjuke"
}
datas << sourceEsObj
}
}
if(dbObj && datas){
def newValMap = getNewField2(datas, dbObj)
if(newValMap){
newValMap.put("id", id)
lastValues << newValMap
}
}
}
statMap.each {source,fieldMap->
println source +" "+fieldMap.size()
fieldMap.each{f,v->
println f+" "+ v
}
println "----------------------------------------"
}
return lastValues
}
def margeValue(lianjiaMap, wiwjMap, anjukeMap){
def lianjisIds = lianjiaMap.keySet()
def wiwjIds = wiwjMap.keySet()
def anjukeIds = anjukeMap.keySet()
def set = new HashSet()
set.addAll(lianjisIds)
set.addAll(wiwjIds)
set.addAll(anjukeIds)
def lastValues = []
def lianjiaUpdateCount = 0
def wiwjUpdateCount = 0
def anjukeUpdateCount = 0
def lianjiaCountMap = [:]
def wiwjCountMap = [:]
def anjukeCountMap = [:]
set.each {id->
BasicDBObject newValMap = new BasicDBObject()
fields.each {field->
def lianjiaValMap = lianjiaMap.get(id)
def wiwjValMap = wiwjMap.get(id)
def anjukeValMap = anjukeMap.get(id)
if(lianjiaValMap && lianjiaValMap.containsKey(field)){
newValMap.put(field, lianjiaValMap.get(field))
lianjiaUpdateCount++
def count = lianjiaCountMap.get(field)
if(!count){
lianjiaCountMap.put(field, 1)
}else{
lianjiaCountMap.put(field, count + 1)
}
}else if(wiwjValMap && wiwjValMap.containsKey(field)){
newValMap.put(field, wiwjValMap.get(field))
wiwjUpdateCount++
def count = wiwjCountMap.get(field)
if(!count){
wiwjCountMap.put(field, 1)
}else{
wiwjCountMap.put(field, count + 1)
}
}else if(anjukeValMap && anjukeValMap.containsKey(field)){
newValMap.put(field, anjukeValMap.get(field))
anjukeUpdateCount++
def count = anjukeCountMap.get(field)
if(!count){
anjukeCountMap.put(field, 1)
}else{
anjukeCountMap.put(field, count + 1)
}
}
}
newValMap.put("id", id)
lastValues << newValMap
}
lianjiaCountMap.each {k,v->
println "[链家] "+k+" "+v
}
wiwjCountMap.each {k,v->
println "[我爱我家] "+k+" "+v
}
anjukeCountMap.each {k,v->
println "[安居客] "+k+" "+v
}
return lastValues
}
def updateDB(margeResults){
for(def mr : margeResults){
def id = mr.get("id")
mr.remove("id")
mr.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',mr))
}
}
def batchUpdateDist(source){
// def rets = findThirdMatchDist(source)
println "[${source}] 符保条件小区量:"+rets.size()
def updateIndex = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
def allUpdateList = new CopyOnWriteArrayList()
def sourceMap = new ConcurrentHashMap()
for(def i=0;i<rets.size();i++) {
final def ret = rets.get(i)
// fixedThreadPool.execute(
// new Runnable() {
// void run() {
try {
def newValMap = running(ret.url, ret.old_match_id)
if(newValMap){
sourceMap.put(ret.old_match_id, newValMap)
allUpdateList << newValMap
updateIndex.addAndGet(1)
}
}catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
// }
// }
// )
}
latch.await()
println "总修改量:"+updateIndex.get()
def countMap = [:]
for(def map : allUpdateList){
map.each{k,v->
def count = countMap.get(k)
if(!count){
countMap.put(k, 1)
}else{
countMap.put(k, count+1)
}
}
}
countMap.each {k,v->
println k +" 更新量:"+v
}
println "---------------------------------------------------------"
return sourceMap
}
def running(url, old_match_id){
def esObj = findCommunities(url)
def dbObj = CommonDatautils.findDistrictById(old_match_id)
if(dbObj){
def newValMap = getNewField(esObj, dbObj)
return newValMap
}
return null
}
def getNewField(esObj, dbObj){
BasicDBObject newValMap = new BasicDBObject()
esObj.each{field, esVal->
def dbVal = dbObj.get(field)
def newVal = null
if(esVal && field=="developer" && esVal!=dbVal){
newVal = FiledStrand.processCompany(esVal)
}
if(esVal && field=="estate_company" && esVal!=dbVal){
newVal = FiledStrand.processCompany(esVal)
}
if(esVal && field=="household_count" && esVal!=dbVal){
newVal = FiledStrand.processHouseholdCount(esVal, dbObj.estate_type2)
}
if(esVal && field=="loudong_count" && esVal!=dbVal){
newVal = FiledStrand.processLoudongCount(esVal)
}
if(esVal && field=="estate_charge" && esVal!=dbVal){
def numVal = FiledStrand.getNumber(esVal)
if(numVal && numVal.toString().toDouble()!=dbVal){
newVal = FiledStrand.processEstateCharge(dbObj.city, numVal, dbObj.estate_type2)
newVal = newVal.toString()
}
}
if(esVal && field=="car_site_count"){
def numVal = FiledStrand.getNumber(esVal)
if(numVal && numVal.toString().toInteger()!=dbVal){
newVal = FiledStrand.processCarSite(numVal)
}
}
if(newVal){
newValMap.put(field, newVal)
}
}
return newValMap
}
def getNewField2(esDatas, dbObj){
BasicDBObject newValMap = new BasicDBObject()
def lastValueMap = [:]
def haveValueMap = [:]
fields.each{field->
for(def esData : esDatas){
def esVal = esData.get(field)
def dbVal = dbObj.get(field)
def newVal = null
if(haveValueMap.containsKey(field)){
continue
}else{
if(lastValueMap.containsKey(field)){
continue
}
if(esVal){
if(field=="estate_charge" || field=="car_site_count") {
esVal = FiledStrand.getNumber(esVal)
}
if(dbVal && esVal.toString()==dbVal.toString()){
haveValueMap.put(field, esVal)
}else{
if(field=="developer" && esVal!=dbVal){
newVal = FiledStrand.processCompany(esVal)
}
if(field=="estate_company" && esVal!=dbVal){
newVal = FiledStrand.processCompany(esVal)
}
if(field=="household_count" && esVal!=dbVal){
newVal = FiledStrand.processHouseholdCount(esVal, dbObj.estate_type2)
}
if(esVal && field=="loudong_count" && esVal!=dbVal){
newVal = FiledStrand.processLoudongCount(esVal)
}
if(field=="estate_charge" && esVal!=dbVal){
def numVal = FiledStrand.getNumber(esVal)
if(numVal && numVal.toString().toDouble()!=dbVal){
newVal = FiledStrand.processEstateCharge(dbObj.city, numVal, dbObj.estate_type2)
newVal = newVal.toString()
}
}
if(field=="car_site_count"){
def numVal = FiledStrand.getNumber(esVal)
if(numVal && numVal.toString().toInteger()!=dbVal){
newVal = FiledStrand.processCarSite(numVal)
}
}
}
}
if(newVal){
lastValueMap.put(field, newVal)
//以下为统计各网站各字段的更新情况
def fieldStatMap = statMap.get(esData.source)
if(!fieldStatMap){
fieldStatMap = [:]
statMap.put(esData.source, fieldStatMap)
}
def count = fieldStatMap.get(field)
if(!count){
fieldStatMap.put(field, 1)
}else{
fieldStatMap.put(field, count + 1)
}
}
}
}
}
return lastValueMap
}
def findThirdMatchDist(source){
def query = new BasicDBObject("source",source).append("old_match_way", 1)
def keys = new BasicDBObject()
keys.put("url", 1)
keys.put("old_match_id", 1)
keys.put("old_match_city", 1)
keys.put("old_match_district", 1)
keys.put("old_match_name", 1)
def idsMap = [:]
def dbcursor = thirdSeaweedMappingColl.find(query, keys)
for(def db : dbcursor){
if(!db.old_match_id && db.old_match_name){
def dbo = CommonDatautils.findDistrict(db.old_match_city, db.old_match_district, db.old_match_name, "district")
if(dbo){
idsMap.put(dbo._id.toString(), db.url)
}
}else{
idsMap.put(db.old_match_id, db.url)
}
}
return idsMap
}
static def findCommunities(url){
if(!url){
return null
}
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("url", url))
def rets = []
def datas = null
try{
datas = ElasticUtils.executeSimpleQuery("192.168.0.206", "community", queryBuilder);
}catch(Exception e){
findCommunities(url)
}
def data = [:]
if(datas) {
for (int k = 0; k < datas.size(); k++) {
def d = datas[k];
data.put("household_count", d.households)
data.put("loudong_count", d.loudong_cnt)
data.put("developer", d.developer)
data.put("estate_company", d.property_company)
data.put("estate_charge", d.property_fee)
data.put("car_site_count", d.parking_cnt)
}
}
return data;
}
}
@@ -26,7 +26,8 @@ class EsMonthHouseStat {
// task.statHouseUpdateCount()
// task.statDistHouse()
task.statRent()
// task.statRent()
task.statOfficeHouse()
}
def statRent(){
@@ -201,6 +202,51 @@ class EsMonthHouseStat {
},0)
}
def statOfficeHouse(){
def citys = CommonDatautils.findAllCity()
def allLists = []
allLists << ["城市", "近一年写字楼出售量", "写字楼量", "近一年出售房源量大于3的写字楼量", "大于3写字楼量占比"]
citys.each{c->
def count = findHouseCountByCat(c, "office_sell",20240101, 20250201)
def districts = CommonDatautils.findAllDistricts(c, "office", ["city", "region", "name", "alias"]);
def disthouseCount = 0
for(def dist : districts){
def alias = []
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
alias = dist.alias
}else{
alias << dist.name
}
def distCount = 0
if(count){
distCount = findHouseCountByCat(dist.city, dist.region, alias, 20240101, 20250201, "office_sell")
if(distCount && distCount>=3){
disthouseCount++
}
}
// println dist.city+" "+dist.region+" "+dist.name+" "+distCount
}
println "disthouseCount:"+disthouseCount
// println districts.size()
def precent = 0
if(districts){
precent = DataHandleUtils.getDoubleRound(2, disthouseCount*100.0/districts.size())
}
def row = [c , count, districts.size(), disthouseCount, precent+"%"]
println row
allLists << row
}
ExcelUtils.write("D:\\数据\\写字楼\\全国写字楼情况占比统计.xlsx", allLists, "写字楼情况")
}
def findHouseCount(def city, def region, def alias, def startDate, def endDate){
def aliasStr = null;
@@ -219,6 +265,23 @@ class EsMonthHouseStat {
return total
}
def findHouseCountByCat(def city, def region, def alias, def startDate, def endDate, cat){
def aliasStr = null;
def total = 0
try{
aliasStr = escapeExprSpecialWord(alias.join("','"))
String sql = "select count(*) as c from latest_seawater where cat='${cat}' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}') "
ElasticUtils.executeSqlQuery(sql,{data->
total = data.c
})
}catch(Exception e){
e.printStackTrace()
}
return total
}
/**
* 转义字符串中的特殊字符
* 仅过滤Lucene regexps, 其他的勿用
@@ -240,6 +303,17 @@ class EsMonthHouseStat {
}
def findHouseCountByCat(def city, def cat, def startDate, def endDate){
String sql = "select count(*) as c from latest_seawater where cat='${cat}' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} "
def total = 0
ElasticUtils.executeSqlQuery(sql,{ data->
total = data.c
})
return total
}