第三方匹配上个月补跑

This commit is contained in:
2024-10-09 13:59:44 +08:00
parent 7cfcc209ec
commit 8167a69773
3 changed files with 126 additions and 10 deletions
@@ -3475,6 +3475,36 @@ class CommonDatautils {
}
return obj;
}
static def queryFirstOrLastRentPrice(city, region, name, sort){
DBObject queryCondition = new BasicDBObject()
queryCondition.put("city", city);
if(region){
queryCondition.put("region", region);
}
if(name){
queryCondition.put("cat", "district");
queryCondition.put("district_name", name)
}else{
if(region){
queryCondition.put("cat", "region")
}else{
queryCondition.put("cat", "city")
}
}
DBCursor cur = SQMETRE_RENT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", sort)).limit(1)
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def obj = 0;
while(cur.hasNext()) {
obj = cur.next()
}
return obj;
}
static def queryCityRegionPrice(city, region, startDate, endDate){
DBObject queryCondition = new BasicDBObject()
@@ -3691,11 +3721,26 @@ class CommonDatautils {
return count;
}
static def findSqmetreRent(def city, def region, def name, def weekend){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("district_name", name)
if(region){
q.put("region", region)
}
if(name){
q.put("district_name", name)
q.put("cat", "district")
}else{
if(!region){
if(city){
q.put("cat", "city")
}else{
q.put("cat", "region")
}
}
}
q.put("weekend", weekend)
def dbcur = SQMETRE_RENT_COLL.find(q)
def dbo = null
@@ -2,20 +2,30 @@ package com.fangjia.script.gj.data_handle
import com.fangjia.core.util.DateUtil
import com.fangjia.script.averager.InsertMonitorCalcCommandBatch
import com.fangjia.script.averager.InsertMonitorSquareRentCalcCommandBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.mongodb.DBCollection
import com.mongodb.MongoClient
class AddCityRegionPrice {
public final static DBCollection COMMAND_COLL = new MongoClient("log.mongo.db.fangjia.com", 52002).getDB('fangjia_log').getCollection('calc_command')
static main(args) {
def task = new AddCityRegionPrice()
// task.addRegionPrice()
// task.addCityprice()
task.addDistrictprice()
// task.addDistrictprice()
// task.addCityRentPrice()
task.addDistrictRentprice()
}
def addRegionPrice(){
InsertMonitorCalcCommandBatch insertBatch = new InsertMonitorCalcCommandBatch()
def citys = CommonDatautils.findAllCity()
def citys = ["文山州"]//CommonDatautils.findAllCity()
def add = 0
for(def c : citys){
def regions = CommonDatautils.getCityRegions(c)
@@ -94,4 +104,58 @@ class AddCityRegionPrice {
}
}
def addCityRentPrice(){
InsertMonitorSquareRentCalcCommandBatch inserBatch = new InsertMonitorSquareRentCalcCommandBatch();
def citys = CommonDatautils.findAllCity()
for(def c : citys){
def startObj = CommonDatautils.queryFirstOrLastRentPrice(c, null, null, 1)
def endObj = CommonDatautils.queryFirstOrLastRentPrice(c, null, null, -1)
if(startObj && endObj){
def months = DateUtils.getBetweenMonth(DateUtil.formatByPattern(startObj.weekend, "yyyyMM"),"202409")
months.each {m->
def lastDay = DateUtils.truncateDay(DateUtils.getLastDayOfMonth(DateUtil.parseDate(m+"01", "yyyyMMdd")))
def obj = CommonDatautils.findSqmetreRent(c, null, null, lastDay)
if(!obj){
inserBatch.saveCityCalcCommand(c, null, null, lastDay, "fangjia-priceanalysis-0")
println "[add] "+c+" "+DateUtil.formatByPattern(lastDay, "yyyy-MM-dd")
}
}
}
}
}
def addDistrictRentprice(){
InsertMonitorSquareRentCalcCommandBatch inserBatch = new InsertMonitorSquareRentCalcCommandBatch();
def citys = CommonDatautils.findAllCity()
for(def c : citys){
def buffer = []
def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "estate_type2"])
for(def d : districts){
def startObj = CommonDatautils.queryFirstOrLastRentPrice(c, d.region, d.name, 1)
// def endObj = CommonDatautils.queryFirstOrLastRentPrice(c, d.region, d.name, -1)
if(startObj){
def months = DateUtils.getBetweenMonth(DateUtil.formatByPattern(startObj.weekend, "yyyyMM"),"202409")
months.each {m->
def lastDay = DateUtils.truncateDay(DateUtils.getLastDayOfMonth(DateUtil.parseDate(m+"01", "yyyyMMdd")))
def obj = CommonDatautils.findSqmetreRent(c, d.region, d.name, lastDay)
if(!obj){
def cmd = inserBatch.saveDistrictCalcCommand(c, d.region, d.name, d.estate_type2, lastDay, "fangjia-priceanalysis-0")
buffer << cmd
if(buffer.size() == 1000){
COMMAND_COLL.insert(buffer)
buffer.clear()
}
println "[add] "+c+" "+ d.name+" "+DateUtil.formatByPattern(lastDay, "yyyy-MM-dd")
}
}
}
}
if(!buffer.isEmpty()){
COMMAND_COLL.insert(buffer)
}
}
}
}
@@ -22,6 +22,7 @@ import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.QueryOperators
import net.sf.json.JSONObject
import org.apache.commons.lang3.SerializationUtils
import org.apache.commons.lang3.StringUtils
import org.apache.tools.ant.util.DateUtils
import org.bson.types.ObjectId
@@ -61,7 +62,7 @@ class DataStandardBatch extends BaseBatch{
private static def districtMap = [:];
private static def limit = 5000;
def static final SHORT_DATE = Integer.parseInt(com.fangjia.script.util.DateUtils.addMonth(com.fangjia.script.util.DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def static final SHORT_DATE = Integer.parseInt(com.fangjia.script.util.DateUtils.addMonth(com.fangjia.script.util.DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6, "无名称":7]
def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋","幸福里","中国房价行情","常州住房"]
@@ -92,7 +93,7 @@ class DataStandardBatch extends BaseBatch{
cityMap.each{city,alias->
println "开始加载【${city}】"
// if(city == "福州"){
// if(city == "上海"){
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"])
districtMap.putAll(detailMap)
// }
@@ -463,7 +464,7 @@ class DataStandardBatch extends BaseBatch{
// }catch (Exception e){
// addr = data.address;
// }
def copyObj = data
def copyObj = SerializationUtils.clone(data)
// copyObj.address = addr
district = getMatchAddressFromMap(copyObj)
if(district){
@@ -492,7 +493,7 @@ class DataStandardBatch extends BaseBatch{
}
if(!district){
def copyObj = data
def copyObj = SerializationUtils.clone(data)
copyObj.address = addr
district = getOpenApiMatch(copyObj)
if(!district && addr){
@@ -906,7 +907,13 @@ class DataStandardBatch extends BaseBatch{
def getOpenApiMatch(def data){
if(StringUtils.isNotEmpty(data.name)){
def jsonObject = OpenApiClient.addressMatch(data.city, data.name, "property", false, data.region, null, "fuzzy", false)
def jsonObject = null
if(DataUtil.isPreciseAddress(data.address)){
jsonObject = OpenApiClient.addressMatch(data.city, data.address, "property", false, data.region, data.name, "fuzzy", false)
}else{
jsonObject = OpenApiClient.addressMatch(data.city, data.name, "property", false, data.region, null, "fuzzy", false)
}
if(jsonObject && jsonObject.credit>0 ){
def dbo = [city:jsonObject.city, region:jsonObject.district, name:jsonObject.name, address:jsonObject.address,estate_type:jsonObject.propertyType, credit:jsonObject.credit]
if(dbo){
@@ -923,7 +930,7 @@ class DataStandardBatch extends BaseBatch{
def findAllNoMatchData(){
DBObject qdata = new BasicDBObject();
qdata.put("s_date", new BasicDBObject('$gte', SHORT_DATE))
// qdata.put("city","福州")
// qdata.put("city","上海")
// qdata.put("source","安居客")
// qdata.put("name",Pattern.compile("龚家宅.*"))
qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false));