Merge branch 'script_gj' into 'master'
第三方匹配上个月补跑 See merge request !973
This commit is contained in:
+47
-2
@@ -3475,6 +3475,36 @@ class CommonDatautils {
|
||||
}
|
||||
return obj;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
static def queryFirstOrLastRentPrice(city, region, name, sort){
|
||||
DBObject queryCondition = new BasicDBObject()
|
||||
queryCondition.put("city", city);
|
||||
if(region){
|
||||
queryCondition.put("region", region);
|
||||
}
|
||||
|
||||
if(name){
|
||||
queryCondition.put("cat", "district");
|
||||
queryCondition.put("district_name", name)
|
||||
}else{
|
||||
if(region){
|
||||
queryCondition.put("cat", "region")
|
||||
}else{
|
||||
queryCondition.put("cat", "city")
|
||||
}
|
||||
}
|
||||
|
||||
DBCursor cur = SQMETRE_RENT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", sort)).limit(1)
|
||||
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def obj = 0;
|
||||
while(cur.hasNext()) {
|
||||
obj = cur.next()
|
||||
}
|
||||
return obj;
|
||||
}
|
||||
|
||||
static def queryCityRegionPrice(city, region, startDate, endDate){
|
||||
DBObject queryCondition = new BasicDBObject()
|
||||
@@ -3691,11 +3721,26 @@ class CommonDatautils {
|
||||
return count;
|
||||
}
|
||||
|
||||
|
||||
static def findSqmetreRent(def city, def region, def name, def weekend){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("district_name", name)
|
||||
if(region){
|
||||
q.put("region", region)
|
||||
}
|
||||
if(name){
|
||||
q.put("district_name", name)
|
||||
q.put("cat", "district")
|
||||
}else{
|
||||
if(!region){
|
||||
if(city){
|
||||
q.put("cat", "city")
|
||||
}else{
|
||||
q.put("cat", "region")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
q.put("weekend", weekend)
|
||||
def dbcur = SQMETRE_RENT_COLL.find(q)
|
||||
def dbo = null
|
||||
|
||||
+66
-2
@@ -2,20 +2,30 @@ package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.averager.InsertMonitorCalcCommandBatch
|
||||
import com.fangjia.script.averager.InsertMonitorSquareRentCalcCommandBatch
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.MongoClient
|
||||
|
||||
class AddCityRegionPrice {
|
||||
|
||||
public final static DBCollection COMMAND_COLL = new MongoClient("log.mongo.db.fangjia.com", 52002).getDB('fangjia_log').getCollection('calc_command')
|
||||
|
||||
static main(args) {
|
||||
def task = new AddCityRegionPrice()
|
||||
// task.addRegionPrice()
|
||||
// task.addCityprice()
|
||||
task.addDistrictprice()
|
||||
// task.addDistrictprice()
|
||||
// task.addCityRentPrice()
|
||||
task.addDistrictRentprice()
|
||||
|
||||
}
|
||||
|
||||
def addRegionPrice(){
|
||||
InsertMonitorCalcCommandBatch insertBatch = new InsertMonitorCalcCommandBatch()
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def citys = ["文山州"]//CommonDatautils.findAllCity()
|
||||
def add = 0
|
||||
for(def c : citys){
|
||||
def regions = CommonDatautils.getCityRegions(c)
|
||||
@@ -94,4 +104,58 @@ class AddCityRegionPrice {
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def addCityRentPrice(){
|
||||
InsertMonitorSquareRentCalcCommandBatch inserBatch = new InsertMonitorSquareRentCalcCommandBatch();
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
for(def c : citys){
|
||||
def startObj = CommonDatautils.queryFirstOrLastRentPrice(c, null, null, 1)
|
||||
def endObj = CommonDatautils.queryFirstOrLastRentPrice(c, null, null, -1)
|
||||
if(startObj && endObj){
|
||||
def months = DateUtils.getBetweenMonth(DateUtil.formatByPattern(startObj.weekend, "yyyyMM"),"202409")
|
||||
months.each {m->
|
||||
def lastDay = DateUtils.truncateDay(DateUtils.getLastDayOfMonth(DateUtil.parseDate(m+"01", "yyyyMMdd")))
|
||||
def obj = CommonDatautils.findSqmetreRent(c, null, null, lastDay)
|
||||
if(!obj){
|
||||
inserBatch.saveCityCalcCommand(c, null, null, lastDay, "fangjia-priceanalysis-0")
|
||||
println "[add] "+c+" "+DateUtil.formatByPattern(lastDay, "yyyy-MM-dd")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def addDistrictRentprice(){
|
||||
InsertMonitorSquareRentCalcCommandBatch inserBatch = new InsertMonitorSquareRentCalcCommandBatch();
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
for(def c : citys){
|
||||
def buffer = []
|
||||
def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "estate_type2"])
|
||||
for(def d : districts){
|
||||
def startObj = CommonDatautils.queryFirstOrLastRentPrice(c, d.region, d.name, 1)
|
||||
// def endObj = CommonDatautils.queryFirstOrLastRentPrice(c, d.region, d.name, -1)
|
||||
if(startObj){
|
||||
def months = DateUtils.getBetweenMonth(DateUtil.formatByPattern(startObj.weekend, "yyyyMM"),"202409")
|
||||
months.each {m->
|
||||
def lastDay = DateUtils.truncateDay(DateUtils.getLastDayOfMonth(DateUtil.parseDate(m+"01", "yyyyMMdd")))
|
||||
def obj = CommonDatautils.findSqmetreRent(c, d.region, d.name, lastDay)
|
||||
if(!obj){
|
||||
def cmd = inserBatch.saveDistrictCalcCommand(c, d.region, d.name, d.estate_type2, lastDay, "fangjia-priceanalysis-0")
|
||||
buffer << cmd
|
||||
if(buffer.size() == 1000){
|
||||
COMMAND_COLL.insert(buffer)
|
||||
buffer.clear()
|
||||
}
|
||||
println "[add] "+c+" "+ d.name+" "+DateUtil.formatByPattern(lastDay, "yyyy-MM-dd")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(!buffer.isEmpty()){
|
||||
COMMAND_COLL.insert(buffer)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+13
-6
@@ -22,6 +22,7 @@ import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.QueryOperators
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.lang3.SerializationUtils
|
||||
import org.apache.commons.lang3.StringUtils
|
||||
import org.apache.tools.ant.util.DateUtils
|
||||
import org.bson.types.ObjectId
|
||||
@@ -61,7 +62,7 @@ class DataStandardBatch extends BaseBatch{
|
||||
private static def districtMap = [:];
|
||||
private static def limit = 5000;
|
||||
|
||||
def static final SHORT_DATE = Integer.parseInt(com.fangjia.script.util.DateUtils.addMonth(com.fangjia.script.util.DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def static final SHORT_DATE = Integer.parseInt(com.fangjia.script.util.DateUtils.addMonth(com.fangjia.script.util.DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
|
||||
def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6, "无名称":7]
|
||||
|
||||
def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋","幸福里","中国房价行情","常州住房"]
|
||||
@@ -92,7 +93,7 @@ class DataStandardBatch extends BaseBatch{
|
||||
|
||||
cityMap.each{city,alias->
|
||||
println "开始加载【${city}】"
|
||||
// if(city == "福州"){
|
||||
// if(city == "上海"){
|
||||
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"])
|
||||
districtMap.putAll(detailMap)
|
||||
// }
|
||||
@@ -463,7 +464,7 @@ class DataStandardBatch extends BaseBatch{
|
||||
// }catch (Exception e){
|
||||
// addr = data.address;
|
||||
// }
|
||||
def copyObj = data
|
||||
def copyObj = SerializationUtils.clone(data)
|
||||
// copyObj.address = addr
|
||||
district = getMatchAddressFromMap(copyObj)
|
||||
if(district){
|
||||
@@ -492,7 +493,7 @@ class DataStandardBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
if(!district){
|
||||
def copyObj = data
|
||||
def copyObj = SerializationUtils.clone(data)
|
||||
copyObj.address = addr
|
||||
district = getOpenApiMatch(copyObj)
|
||||
if(!district && addr){
|
||||
@@ -906,7 +907,13 @@ class DataStandardBatch extends BaseBatch{
|
||||
|
||||
def getOpenApiMatch(def data){
|
||||
if(StringUtils.isNotEmpty(data.name)){
|
||||
def jsonObject = OpenApiClient.addressMatch(data.city, data.name, "property", false, data.region, null, "fuzzy", false)
|
||||
def jsonObject = null
|
||||
if(DataUtil.isPreciseAddress(data.address)){
|
||||
jsonObject = OpenApiClient.addressMatch(data.city, data.address, "property", false, data.region, data.name, "fuzzy", false)
|
||||
}else{
|
||||
jsonObject = OpenApiClient.addressMatch(data.city, data.name, "property", false, data.region, null, "fuzzy", false)
|
||||
}
|
||||
|
||||
if(jsonObject && jsonObject.credit>0 ){
|
||||
def dbo = [city:jsonObject.city, region:jsonObject.district, name:jsonObject.name, address:jsonObject.address,estate_type:jsonObject.propertyType, credit:jsonObject.credit]
|
||||
if(dbo){
|
||||
@@ -923,7 +930,7 @@ class DataStandardBatch extends BaseBatch{
|
||||
def findAllNoMatchData(){
|
||||
DBObject qdata = new BasicDBObject();
|
||||
qdata.put("s_date", new BasicDBObject('$gte', SHORT_DATE))
|
||||
// qdata.put("city","福州")
|
||||
// qdata.put("city","上海")
|
||||
// qdata.put("source","安居客")
|
||||
// qdata.put("name",Pattern.compile("龚家宅.*"))
|
||||
qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false));
|
||||
|
||||
Reference in New Issue
Block a user