Merge remote-tracking branch 'origin/master'
This commit is contained in:
+9
-4
@@ -5137,17 +5137,22 @@ class CommonDatautils {
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def townMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def distName = null
|
||||
if(d.mzb_district_name){
|
||||
def list = townMap.get(d.mzb_district_name)
|
||||
distName = d.mzb_district_name
|
||||
}else{
|
||||
distName = d.district
|
||||
}
|
||||
if(distName){
|
||||
def list = townMap.get(distName)
|
||||
if(!list){
|
||||
list = []
|
||||
}
|
||||
list << d
|
||||
townMap.put(d.mzb_district_name, list)
|
||||
townMap.put(distName, list)
|
||||
}
|
||||
}
|
||||
|
||||
println "[${city}] 乡镇街道 加载完成 "+new Date().format("yyyy-MM-dd HH:mm:ss")
|
||||
// println "[${city}] 乡镇街道 加载完成 "+new Date().format("yyyy-MM-dd HH:mm:ss")
|
||||
return townMap
|
||||
}
|
||||
|
||||
|
||||
@@ -909,14 +909,18 @@ class FiledStrand {
|
||||
}
|
||||
|
||||
static def getNumber(def input) {
|
||||
Pattern pattern = Pattern.compile("\\d+(\\.\\d+)?");
|
||||
Matcher matcher = pattern.matcher(input);
|
||||
List<String> list = new ArrayList();
|
||||
while (matcher.find()) {
|
||||
list.add(matcher.group());
|
||||
}
|
||||
if(list.size()>0){
|
||||
return list.get(0);
|
||||
try{
|
||||
Pattern pattern = Pattern.compile("\\d+(\\.\\d+)?");
|
||||
Matcher matcher = pattern.matcher(input);
|
||||
List<String> list = new ArrayList();
|
||||
while (matcher.find()) {
|
||||
list.add(matcher.group());
|
||||
}
|
||||
if(list.size()>0){
|
||||
return list.get(0);
|
||||
}
|
||||
}catch(Exception e){
|
||||
println input+" error"
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
+413
@@ -0,0 +1,413 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.FiledStrand
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/**
|
||||
* 根据链家数据修改小区库中字段
|
||||
* 使用到中间映射表
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class UpdateDistrictField extends BaseBatch{
|
||||
|
||||
final DBCollection thirdSeaweedMappingColl = MongoUtils.getCollectionByName("fangjia_craw", "third_seaweed_mapping", "192.168.0.235", 27018)
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
def fields = ["developer","estate_company","household_count","loudong_count","estate_charge","car_site_count"]
|
||||
|
||||
def sources = ["链家", "我爱我家", "安居客"]
|
||||
|
||||
def statMap = [:]
|
||||
|
||||
static main(args) {
|
||||
runBatch(new UpdateDistrictField(), args)
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
def sourceMap = [:]
|
||||
sources.each {source->
|
||||
def valMap = findThirdMatchDist(source)
|
||||
println source +" "+valMap.size()
|
||||
if(valMap){
|
||||
sourceMap.put(source, valMap)
|
||||
}
|
||||
}
|
||||
def margeResults = megerDatas(sourceMap)
|
||||
// updateDB(margeResults)
|
||||
}
|
||||
|
||||
def megerDatas(sourceMap){
|
||||
def set = new HashSet()
|
||||
sources.each {source->
|
||||
def valMap = sourceMap.get(source)
|
||||
set.addAll(valMap.keySet())
|
||||
}
|
||||
def lastValues = []
|
||||
set.each {id->
|
||||
|
||||
def dbObj = CommonDatautils.findDistrictById(id)
|
||||
|
||||
def datas = []
|
||||
sources.each {source->
|
||||
def valMap = sourceMap.get(source)
|
||||
def sourceEsObj = findCommunities(valMap.get(id))
|
||||
if(sourceEsObj){
|
||||
if(source == "链家"){
|
||||
sourceEsObj.source = "lianjia"
|
||||
}else if(source == "我爱我家"){
|
||||
sourceEsObj.source = "wiwj"
|
||||
}else if(source == "安居客"){
|
||||
sourceEsObj.source = "anjuke"
|
||||
}
|
||||
datas << sourceEsObj
|
||||
}
|
||||
}
|
||||
if(dbObj && datas){
|
||||
def newValMap = getNewField2(datas, dbObj)
|
||||
if(newValMap){
|
||||
newValMap.put("id", id)
|
||||
lastValues << newValMap
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
statMap.each {source,fieldMap->
|
||||
println source +" "+fieldMap.size()
|
||||
fieldMap.each{f,v->
|
||||
println f+" "+ v
|
||||
}
|
||||
println "----------------------------------------"
|
||||
}
|
||||
return lastValues
|
||||
}
|
||||
|
||||
|
||||
def margeValue(lianjiaMap, wiwjMap, anjukeMap){
|
||||
def lianjisIds = lianjiaMap.keySet()
|
||||
def wiwjIds = wiwjMap.keySet()
|
||||
def anjukeIds = anjukeMap.keySet()
|
||||
def set = new HashSet()
|
||||
set.addAll(lianjisIds)
|
||||
set.addAll(wiwjIds)
|
||||
set.addAll(anjukeIds)
|
||||
|
||||
def lastValues = []
|
||||
def lianjiaUpdateCount = 0
|
||||
def wiwjUpdateCount = 0
|
||||
def anjukeUpdateCount = 0
|
||||
def lianjiaCountMap = [:]
|
||||
def wiwjCountMap = [:]
|
||||
def anjukeCountMap = [:]
|
||||
set.each {id->
|
||||
BasicDBObject newValMap = new BasicDBObject()
|
||||
fields.each {field->
|
||||
def lianjiaValMap = lianjiaMap.get(id)
|
||||
def wiwjValMap = wiwjMap.get(id)
|
||||
def anjukeValMap = anjukeMap.get(id)
|
||||
if(lianjiaValMap && lianjiaValMap.containsKey(field)){
|
||||
newValMap.put(field, lianjiaValMap.get(field))
|
||||
lianjiaUpdateCount++
|
||||
def count = lianjiaCountMap.get(field)
|
||||
if(!count){
|
||||
lianjiaCountMap.put(field, 1)
|
||||
}else{
|
||||
lianjiaCountMap.put(field, count + 1)
|
||||
}
|
||||
}else if(wiwjValMap && wiwjValMap.containsKey(field)){
|
||||
newValMap.put(field, wiwjValMap.get(field))
|
||||
wiwjUpdateCount++
|
||||
def count = wiwjCountMap.get(field)
|
||||
if(!count){
|
||||
wiwjCountMap.put(field, 1)
|
||||
}else{
|
||||
wiwjCountMap.put(field, count + 1)
|
||||
}
|
||||
}else if(anjukeValMap && anjukeValMap.containsKey(field)){
|
||||
newValMap.put(field, anjukeValMap.get(field))
|
||||
anjukeUpdateCount++
|
||||
def count = anjukeCountMap.get(field)
|
||||
if(!count){
|
||||
anjukeCountMap.put(field, 1)
|
||||
}else{
|
||||
anjukeCountMap.put(field, count + 1)
|
||||
}
|
||||
}
|
||||
}
|
||||
newValMap.put("id", id)
|
||||
lastValues << newValMap
|
||||
}
|
||||
|
||||
lianjiaCountMap.each {k,v->
|
||||
println "[链家] "+k+" "+v
|
||||
}
|
||||
|
||||
wiwjCountMap.each {k,v->
|
||||
println "[我爱我家] "+k+" "+v
|
||||
}
|
||||
|
||||
anjukeCountMap.each {k,v->
|
||||
println "[安居客] "+k+" "+v
|
||||
}
|
||||
return lastValues
|
||||
}
|
||||
|
||||
def updateDB(margeResults){
|
||||
for(def mr : margeResults){
|
||||
def id = mr.get("id")
|
||||
mr.remove("id")
|
||||
mr.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',mr))
|
||||
}
|
||||
}
|
||||
|
||||
def batchUpdateDist(source){
|
||||
// def rets = findThirdMatchDist(source)
|
||||
println "[${source}] 符保条件小区量:"+rets.size()
|
||||
def updateIndex = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(rets.size())
|
||||
def allUpdateList = new CopyOnWriteArrayList()
|
||||
|
||||
def sourceMap = new ConcurrentHashMap()
|
||||
|
||||
for(def i=0;i<rets.size();i++) {
|
||||
final def ret = rets.get(i)
|
||||
// fixedThreadPool.execute(
|
||||
// new Runnable() {
|
||||
// void run() {
|
||||
try {
|
||||
def newValMap = running(ret.url, ret.old_match_id)
|
||||
if(newValMap){
|
||||
sourceMap.put(ret.old_match_id, newValMap)
|
||||
allUpdateList << newValMap
|
||||
updateIndex.addAndGet(1)
|
||||
}
|
||||
}catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
// }
|
||||
// }
|
||||
// )
|
||||
}
|
||||
latch.await()
|
||||
println "总修改量:"+updateIndex.get()
|
||||
def countMap = [:]
|
||||
for(def map : allUpdateList){
|
||||
map.each{k,v->
|
||||
def count = countMap.get(k)
|
||||
if(!count){
|
||||
countMap.put(k, 1)
|
||||
}else{
|
||||
countMap.put(k, count+1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
countMap.each {k,v->
|
||||
println k +" 更新量:"+v
|
||||
}
|
||||
|
||||
println "---------------------------------------------------------"
|
||||
return sourceMap
|
||||
}
|
||||
|
||||
def running(url, old_match_id){
|
||||
def esObj = findCommunities(url)
|
||||
def dbObj = CommonDatautils.findDistrictById(old_match_id)
|
||||
if(dbObj){
|
||||
def newValMap = getNewField(esObj, dbObj)
|
||||
return newValMap
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def getNewField(esObj, dbObj){
|
||||
BasicDBObject newValMap = new BasicDBObject()
|
||||
esObj.each{field, esVal->
|
||||
def dbVal = dbObj.get(field)
|
||||
def newVal = null
|
||||
if(esVal && field=="developer" && esVal!=dbVal){
|
||||
newVal = FiledStrand.processCompany(esVal)
|
||||
|
||||
}
|
||||
if(esVal && field=="estate_company" && esVal!=dbVal){
|
||||
newVal = FiledStrand.processCompany(esVal)
|
||||
}
|
||||
|
||||
if(esVal && field=="household_count" && esVal!=dbVal){
|
||||
newVal = FiledStrand.processHouseholdCount(esVal, dbObj.estate_type2)
|
||||
}
|
||||
|
||||
if(esVal && field=="loudong_count" && esVal!=dbVal){
|
||||
newVal = FiledStrand.processLoudongCount(esVal)
|
||||
}
|
||||
|
||||
if(esVal && field=="estate_charge" && esVal!=dbVal){
|
||||
def numVal = FiledStrand.getNumber(esVal)
|
||||
if(numVal && numVal.toString().toDouble()!=dbVal){
|
||||
newVal = FiledStrand.processEstateCharge(dbObj.city, numVal, dbObj.estate_type2)
|
||||
newVal = newVal.toString()
|
||||
}
|
||||
}
|
||||
if(esVal && field=="car_site_count"){
|
||||
def numVal = FiledStrand.getNumber(esVal)
|
||||
if(numVal && numVal.toString().toInteger()!=dbVal){
|
||||
newVal = FiledStrand.processCarSite(numVal)
|
||||
}
|
||||
}
|
||||
if(newVal){
|
||||
newValMap.put(field, newVal)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return newValMap
|
||||
}
|
||||
|
||||
def getNewField2(esDatas, dbObj){
|
||||
BasicDBObject newValMap = new BasicDBObject()
|
||||
def lastValueMap = [:]
|
||||
def haveValueMap = [:]
|
||||
fields.each{field->
|
||||
for(def esData : esDatas){
|
||||
def esVal = esData.get(field)
|
||||
def dbVal = dbObj.get(field)
|
||||
def newVal = null
|
||||
if(haveValueMap.containsKey(field)){
|
||||
continue
|
||||
}else{
|
||||
if(lastValueMap.containsKey(field)){
|
||||
continue
|
||||
}
|
||||
if(esVal){
|
||||
|
||||
if(field=="estate_charge" || field=="car_site_count") {
|
||||
esVal = FiledStrand.getNumber(esVal)
|
||||
}
|
||||
if(dbVal && esVal.toString()==dbVal.toString()){
|
||||
haveValueMap.put(field, esVal)
|
||||
}else{
|
||||
if(field=="developer" && esVal!=dbVal){
|
||||
newVal = FiledStrand.processCompany(esVal)
|
||||
}
|
||||
|
||||
if(field=="estate_company" && esVal!=dbVal){
|
||||
newVal = FiledStrand.processCompany(esVal)
|
||||
}
|
||||
|
||||
if(field=="household_count" && esVal!=dbVal){
|
||||
newVal = FiledStrand.processHouseholdCount(esVal, dbObj.estate_type2)
|
||||
}
|
||||
|
||||
if(esVal && field=="loudong_count" && esVal!=dbVal){
|
||||
newVal = FiledStrand.processLoudongCount(esVal)
|
||||
}
|
||||
|
||||
if(field=="estate_charge" && esVal!=dbVal){
|
||||
def numVal = FiledStrand.getNumber(esVal)
|
||||
if(numVal && numVal.toString().toDouble()!=dbVal){
|
||||
newVal = FiledStrand.processEstateCharge(dbObj.city, numVal, dbObj.estate_type2)
|
||||
newVal = newVal.toString()
|
||||
}
|
||||
}
|
||||
if(field=="car_site_count"){
|
||||
def numVal = FiledStrand.getNumber(esVal)
|
||||
if(numVal && numVal.toString().toInteger()!=dbVal){
|
||||
newVal = FiledStrand.processCarSite(numVal)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if(newVal){
|
||||
lastValueMap.put(field, newVal)
|
||||
|
||||
//以下为统计各网站各字段的更新情况
|
||||
def fieldStatMap = statMap.get(esData.source)
|
||||
if(!fieldStatMap){
|
||||
fieldStatMap = [:]
|
||||
statMap.put(esData.source, fieldStatMap)
|
||||
}
|
||||
def count = fieldStatMap.get(field)
|
||||
if(!count){
|
||||
fieldStatMap.put(field, 1)
|
||||
}else{
|
||||
fieldStatMap.put(field, count + 1)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return lastValueMap
|
||||
}
|
||||
|
||||
def findThirdMatchDist(source){
|
||||
def query = new BasicDBObject("source",source).append("old_match_way", 1)
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("url", 1)
|
||||
keys.put("old_match_id", 1)
|
||||
keys.put("old_match_city", 1)
|
||||
keys.put("old_match_district", 1)
|
||||
keys.put("old_match_name", 1)
|
||||
def idsMap = [:]
|
||||
def dbcursor = thirdSeaweedMappingColl.find(query, keys)
|
||||
for(def db : dbcursor){
|
||||
if(!db.old_match_id && db.old_match_name){
|
||||
def dbo = CommonDatautils.findDistrict(db.old_match_city, db.old_match_district, db.old_match_name, "district")
|
||||
if(dbo){
|
||||
idsMap.put(dbo._id.toString(), db.url)
|
||||
}
|
||||
}else{
|
||||
idsMap.put(db.old_match_id, db.url)
|
||||
}
|
||||
|
||||
}
|
||||
return idsMap
|
||||
}
|
||||
|
||||
static def findCommunities(url){
|
||||
if(!url){
|
||||
return null
|
||||
}
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("url", url))
|
||||
def rets = []
|
||||
def datas = null
|
||||
try{
|
||||
datas = ElasticUtils.executeSimpleQuery("192.168.0.206", "community", queryBuilder);
|
||||
}catch(Exception e){
|
||||
findCommunities(url)
|
||||
}
|
||||
def data = [:]
|
||||
if(datas) {
|
||||
for (int k = 0; k < datas.size(); k++) {
|
||||
def d = datas[k];
|
||||
data.put("household_count", d.households)
|
||||
data.put("loudong_count", d.loudong_cnt)
|
||||
data.put("developer", d.developer)
|
||||
data.put("estate_company", d.property_company)
|
||||
data.put("estate_charge", d.property_fee)
|
||||
data.put("car_site_count", d.parking_cnt)
|
||||
}
|
||||
}
|
||||
return data;
|
||||
}
|
||||
}
|
||||
+75
-1
@@ -26,7 +26,8 @@ class EsMonthHouseStat {
|
||||
// task.statHouseUpdateCount()
|
||||
|
||||
// task.statDistHouse()
|
||||
task.statRent()
|
||||
// task.statRent()
|
||||
task.statOfficeHouse()
|
||||
}
|
||||
|
||||
def statRent(){
|
||||
@@ -201,6 +202,51 @@ class EsMonthHouseStat {
|
||||
},0)
|
||||
}
|
||||
|
||||
def statOfficeHouse(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def allLists = []
|
||||
allLists << ["城市", "近一年写字楼出售量", "写字楼量", "近一年出售房源量大于3的写字楼量", "大于3写字楼量占比"]
|
||||
citys.each{c->
|
||||
def count = findHouseCountByCat(c, "office_sell",20240101, 20250201)
|
||||
|
||||
def districts = CommonDatautils.findAllDistricts(c, "office", ["city", "region", "name", "alias"]);
|
||||
def disthouseCount = 0
|
||||
for(def dist : districts){
|
||||
def alias = []
|
||||
if(dist.alias){
|
||||
if(!dist.alias.contains(dist.name)){
|
||||
dist.alias << dist.name
|
||||
}
|
||||
alias = dist.alias
|
||||
}else{
|
||||
alias << dist.name
|
||||
}
|
||||
|
||||
def distCount = 0
|
||||
if(count){
|
||||
distCount = findHouseCountByCat(dist.city, dist.region, alias, 20240101, 20250201, "office_sell")
|
||||
if(distCount && distCount>=3){
|
||||
disthouseCount++
|
||||
}
|
||||
}
|
||||
// println dist.city+" "+dist.region+" "+dist.name+" "+distCount
|
||||
}
|
||||
|
||||
println "disthouseCount:"+disthouseCount
|
||||
// println districts.size()
|
||||
def precent = 0
|
||||
if(districts){
|
||||
precent = DataHandleUtils.getDoubleRound(2, disthouseCount*100.0/districts.size())
|
||||
}
|
||||
def row = [c , count, districts.size(), disthouseCount, precent+"%"]
|
||||
println row
|
||||
allLists << row
|
||||
}
|
||||
|
||||
ExcelUtils.write("D:\\数据\\写字楼\\全国写字楼情况占比统计.xlsx", allLists, "写字楼情况")
|
||||
|
||||
}
|
||||
|
||||
|
||||
def findHouseCount(def city, def region, def alias, def startDate, def endDate){
|
||||
def aliasStr = null;
|
||||
@@ -219,6 +265,23 @@ class EsMonthHouseStat {
|
||||
return total
|
||||
}
|
||||
|
||||
def findHouseCountByCat(def city, def region, def alias, def startDate, def endDate, cat){
|
||||
def aliasStr = null;
|
||||
def total = 0
|
||||
try{
|
||||
aliasStr = escapeExprSpecialWord(alias.join("','"))
|
||||
String sql = "select count(*) as c from latest_seawater where cat='${cat}' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}') "
|
||||
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
total = data.c
|
||||
})
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
|
||||
return total
|
||||
}
|
||||
|
||||
/**
|
||||
* 转义字符串中的特殊字符
|
||||
* 仅过滤Lucene regexps, 其他的勿用
|
||||
@@ -240,6 +303,17 @@ class EsMonthHouseStat {
|
||||
}
|
||||
|
||||
|
||||
def findHouseCountByCat(def city, def cat, def startDate, def endDate){
|
||||
|
||||
String sql = "select count(*) as c from latest_seawater where cat='${cat}' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} "
|
||||
def total = 0
|
||||
ElasticUtils.executeSqlQuery(sql,{ data->
|
||||
total = data.c
|
||||
})
|
||||
return total
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user