diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index f710a70fe..94668a4df 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -3475,6 +3475,36 @@ class CommonDatautils { } return obj; } + + + + + static def queryFirstOrLastRentPrice(city, region, name, sort){ + DBObject queryCondition = new BasicDBObject() + queryCondition.put("city", city); + if(region){ + queryCondition.put("region", region); + } + + if(name){ + queryCondition.put("cat", "district"); + queryCondition.put("district_name", name) + }else{ + if(region){ + queryCondition.put("cat", "region") + }else{ + queryCondition.put("cat", "city") + } + } + + DBCursor cur = SQMETRE_RENT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", sort)).limit(1) + cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT) + def obj = 0; + while(cur.hasNext()) { + obj = cur.next() + } + return obj; + } static def queryCityRegionPrice(city, region, startDate, endDate){ DBObject queryCondition = new BasicDBObject() @@ -3691,11 +3721,26 @@ class CommonDatautils { return count; } + static def findSqmetreRent(def city, def region, def name, def weekend){ def q = new BasicDBObject() q.put("city", city) - q.put("region", region) - q.put("district_name", name) + if(region){ + q.put("region", region) + } + if(name){ + q.put("district_name", name) + q.put("cat", "district") + }else{ + if(!region){ + if(city){ + q.put("cat", "city") + }else{ + q.put("cat", "region") + } + } + } + q.put("weekend", weekend) def dbcur = SQMETRE_RENT_COLL.find(q) def dbo = null diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy index 667f06d6d..cc42938d7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy @@ -2,20 +2,30 @@ package com.fangjia.script.gj.data_handle import com.fangjia.core.util.DateUtil import com.fangjia.script.averager.InsertMonitorCalcCommandBatch +import com.fangjia.script.averager.InsertMonitorSquareRentCalcCommandBatch import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.mongodb.DBCollection +import com.mongodb.MongoClient class AddCityRegionPrice { + + public final static DBCollection COMMAND_COLL = new MongoClient("log.mongo.db.fangjia.com", 52002).getDB('fangjia_log').getCollection('calc_command') + static main(args) { def task = new AddCityRegionPrice() // task.addRegionPrice() // task.addCityprice() - task.addDistrictprice() +// task.addDistrictprice() +// task.addCityRentPrice() + task.addDistrictRentprice() + } def addRegionPrice(){ InsertMonitorCalcCommandBatch insertBatch = new InsertMonitorCalcCommandBatch() - def citys = CommonDatautils.findAllCity() + def citys = ["文山州"]//CommonDatautils.findAllCity() def add = 0 for(def c : citys){ def regions = CommonDatautils.getCityRegions(c) @@ -94,4 +104,58 @@ class AddCityRegionPrice { } } + + def addCityRentPrice(){ + InsertMonitorSquareRentCalcCommandBatch inserBatch = new InsertMonitorSquareRentCalcCommandBatch(); + def citys = CommonDatautils.findAllCity() + for(def c : citys){ + def startObj = CommonDatautils.queryFirstOrLastRentPrice(c, null, null, 1) + def endObj = CommonDatautils.queryFirstOrLastRentPrice(c, null, null, -1) + if(startObj && endObj){ + def months = DateUtils.getBetweenMonth(DateUtil.formatByPattern(startObj.weekend, "yyyyMM"),"202409") + months.each {m-> + def lastDay = DateUtils.truncateDay(DateUtils.getLastDayOfMonth(DateUtil.parseDate(m+"01", "yyyyMMdd"))) + def obj = CommonDatautils.findSqmetreRent(c, null, null, lastDay) + if(!obj){ + inserBatch.saveCityCalcCommand(c, null, null, lastDay, "fangjia-priceanalysis-0") + println "[add] "+c+" "+DateUtil.formatByPattern(lastDay, "yyyy-MM-dd") + } + } + } + } + } + + def addDistrictRentprice(){ + InsertMonitorSquareRentCalcCommandBatch inserBatch = new InsertMonitorSquareRentCalcCommandBatch(); + def citys = CommonDatautils.findAllCity() + for(def c : citys){ + def buffer = [] + def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "estate_type2"]) + for(def d : districts){ + def startObj = CommonDatautils.queryFirstOrLastRentPrice(c, d.region, d.name, 1) +// def endObj = CommonDatautils.queryFirstOrLastRentPrice(c, d.region, d.name, -1) + if(startObj){ + def months = DateUtils.getBetweenMonth(DateUtil.formatByPattern(startObj.weekend, "yyyyMM"),"202409") + months.each {m-> + def lastDay = DateUtils.truncateDay(DateUtils.getLastDayOfMonth(DateUtil.parseDate(m+"01", "yyyyMMdd"))) + def obj = CommonDatautils.findSqmetreRent(c, d.region, d.name, lastDay) + if(!obj){ + def cmd = inserBatch.saveDistrictCalcCommand(c, d.region, d.name, d.estate_type2, lastDay, "fangjia-priceanalysis-0") + buffer << cmd + if(buffer.size() == 1000){ + COMMAND_COLL.insert(buffer) + buffer.clear() + } + println "[add] "+c+" "+ d.name+" "+DateUtil.formatByPattern(lastDay, "yyyy-MM-dd") + } + } + } + } + + if(!buffer.isEmpty()){ + COMMAND_COLL.insert(buffer) + } + + } + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy index f1df981ef..1d63a790f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy @@ -22,6 +22,7 @@ import com.mongodb.DBCollection import com.mongodb.DBObject import com.mongodb.QueryOperators import net.sf.json.JSONObject +import org.apache.commons.lang3.SerializationUtils import org.apache.commons.lang3.StringUtils import org.apache.tools.ant.util.DateUtils import org.bson.types.ObjectId @@ -61,7 +62,7 @@ class DataStandardBatch extends BaseBatch{ private static def districtMap = [:]; private static def limit = 5000; - def static final SHORT_DATE = Integer.parseInt(com.fangjia.script.util.DateUtils.addMonth(com.fangjia.script.util.DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def static final SHORT_DATE = Integer.parseInt(com.fangjia.script.util.DateUtils.addMonth(com.fangjia.script.util.DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6, "无名称":7] def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋","幸福里","中国房价行情","常州住房"] @@ -92,7 +93,7 @@ class DataStandardBatch extends BaseBatch{ cityMap.each{city,alias-> println "开始加载【${city}】" -// if(city == "福州"){ +// if(city == "上海"){ def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"]) districtMap.putAll(detailMap) // } @@ -463,7 +464,7 @@ class DataStandardBatch extends BaseBatch{ // }catch (Exception e){ // addr = data.address; // } - def copyObj = data + def copyObj = SerializationUtils.clone(data) // copyObj.address = addr district = getMatchAddressFromMap(copyObj) if(district){ @@ -492,7 +493,7 @@ class DataStandardBatch extends BaseBatch{ } if(!district){ - def copyObj = data + def copyObj = SerializationUtils.clone(data) copyObj.address = addr district = getOpenApiMatch(copyObj) if(!district && addr){ @@ -906,7 +907,13 @@ class DataStandardBatch extends BaseBatch{ def getOpenApiMatch(def data){ if(StringUtils.isNotEmpty(data.name)){ - def jsonObject = OpenApiClient.addressMatch(data.city, data.name, "property", false, data.region, null, "fuzzy", false) + def jsonObject = null + if(DataUtil.isPreciseAddress(data.address)){ + jsonObject = OpenApiClient.addressMatch(data.city, data.address, "property", false, data.region, data.name, "fuzzy", false) + }else{ + jsonObject = OpenApiClient.addressMatch(data.city, data.name, "property", false, data.region, null, "fuzzy", false) + } + if(jsonObject && jsonObject.credit>0 ){ def dbo = [city:jsonObject.city, region:jsonObject.district, name:jsonObject.name, address:jsonObject.address,estate_type:jsonObject.propertyType, credit:jsonObject.credit] if(dbo){ @@ -923,7 +930,7 @@ class DataStandardBatch extends BaseBatch{ def findAllNoMatchData(){ DBObject qdata = new BasicDBObject(); qdata.put("s_date", new BasicDBObject('$gte', SHORT_DATE)) -// qdata.put("city","福州") +// qdata.put("city","上海") // qdata.put("source","安居客") // qdata.put("name",Pattern.compile("龚家宅.*")) qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false));