diff --git a/projects/fangjia-coresite/src/main/webapp/trend/trend.ftl b/projects/fangjia-coresite/src/main/webapp/trend/trend.ftl
index 57168fb3d..33108e5f4 100644
--- a/projects/fangjia-coresite/src/main/webapp/trend/trend.ftl
+++ b/projects/fangjia-coresite/src/main/webapp/trend/trend.ftl
@@ -52,18 +52,18 @@
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy
index f710a70fe..94668a4df 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy
@@ -3475,6 +3475,36 @@ class CommonDatautils {
}
return obj;
}
+
+
+
+
+ static def queryFirstOrLastRentPrice(city, region, name, sort){
+ DBObject queryCondition = new BasicDBObject()
+ queryCondition.put("city", city);
+ if(region){
+ queryCondition.put("region", region);
+ }
+
+ if(name){
+ queryCondition.put("cat", "district");
+ queryCondition.put("district_name", name)
+ }else{
+ if(region){
+ queryCondition.put("cat", "region")
+ }else{
+ queryCondition.put("cat", "city")
+ }
+ }
+
+ DBCursor cur = SQMETRE_RENT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", sort)).limit(1)
+ cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
+ def obj = 0;
+ while(cur.hasNext()) {
+ obj = cur.next()
+ }
+ return obj;
+ }
static def queryCityRegionPrice(city, region, startDate, endDate){
DBObject queryCondition = new BasicDBObject()
@@ -3691,11 +3721,26 @@ class CommonDatautils {
return count;
}
+
static def findSqmetreRent(def city, def region, def name, def weekend){
def q = new BasicDBObject()
q.put("city", city)
- q.put("region", region)
- q.put("district_name", name)
+ if(region){
+ q.put("region", region)
+ }
+ if(name){
+ q.put("district_name", name)
+ q.put("cat", "district")
+ }else{
+ if(!region){
+ if(city){
+ q.put("cat", "city")
+ }else{
+ q.put("cat", "region")
+ }
+ }
+ }
+
q.put("weekend", weekend)
def dbcur = SQMETRE_RENT_COLL.find(q)
def dbo = null
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy
index 667f06d6d..cc42938d7 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy
@@ -2,20 +2,30 @@ package com.fangjia.script.gj.data_handle
import com.fangjia.core.util.DateUtil
import com.fangjia.script.averager.InsertMonitorCalcCommandBatch
+import com.fangjia.script.averager.InsertMonitorSquareRentCalcCommandBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.CommonUtils
+import com.fangjia.script.util.DateUtils
+import com.mongodb.DBCollection
+import com.mongodb.MongoClient
class AddCityRegionPrice {
+
+ public final static DBCollection COMMAND_COLL = new MongoClient("log.mongo.db.fangjia.com", 52002).getDB('fangjia_log').getCollection('calc_command')
+
static main(args) {
def task = new AddCityRegionPrice()
// task.addRegionPrice()
// task.addCityprice()
- task.addDistrictprice()
+// task.addDistrictprice()
+// task.addCityRentPrice()
+ task.addDistrictRentprice()
+
}
def addRegionPrice(){
InsertMonitorCalcCommandBatch insertBatch = new InsertMonitorCalcCommandBatch()
- def citys = CommonDatautils.findAllCity()
+ def citys = ["ÎÄɽÖÝ"]//CommonDatautils.findAllCity()
def add = 0
for(def c : citys){
def regions = CommonDatautils.getCityRegions(c)
@@ -94,4 +104,58 @@ class AddCityRegionPrice {
}
}
+
+ def addCityRentPrice(){
+ InsertMonitorSquareRentCalcCommandBatch inserBatch = new InsertMonitorSquareRentCalcCommandBatch();
+ def citys = CommonDatautils.findAllCity()
+ for(def c : citys){
+ def startObj = CommonDatautils.queryFirstOrLastRentPrice(c, null, null, 1)
+ def endObj = CommonDatautils.queryFirstOrLastRentPrice(c, null, null, -1)
+ if(startObj && endObj){
+ def months = DateUtils.getBetweenMonth(DateUtil.formatByPattern(startObj.weekend, "yyyyMM"),"202409")
+ months.each {m->
+ def lastDay = DateUtils.truncateDay(DateUtils.getLastDayOfMonth(DateUtil.parseDate(m+"01", "yyyyMMdd")))
+ def obj = CommonDatautils.findSqmetreRent(c, null, null, lastDay)
+ if(!obj){
+ inserBatch.saveCityCalcCommand(c, null, null, lastDay, "fangjia-priceanalysis-0")
+ println "[add] "+c+" "+DateUtil.formatByPattern(lastDay, "yyyy-MM-dd")
+ }
+ }
+ }
+ }
+ }
+
+ def addDistrictRentprice(){
+ InsertMonitorSquareRentCalcCommandBatch inserBatch = new InsertMonitorSquareRentCalcCommandBatch();
+ def citys = CommonDatautils.findAllCity()
+ for(def c : citys){
+ def buffer = []
+ def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "estate_type2"])
+ for(def d : districts){
+ def startObj = CommonDatautils.queryFirstOrLastRentPrice(c, d.region, d.name, 1)
+// def endObj = CommonDatautils.queryFirstOrLastRentPrice(c, d.region, d.name, -1)
+ if(startObj){
+ def months = DateUtils.getBetweenMonth(DateUtil.formatByPattern(startObj.weekend, "yyyyMM"),"202409")
+ months.each {m->
+ def lastDay = DateUtils.truncateDay(DateUtils.getLastDayOfMonth(DateUtil.parseDate(m+"01", "yyyyMMdd")))
+ def obj = CommonDatautils.findSqmetreRent(c, d.region, d.name, lastDay)
+ if(!obj){
+ def cmd = inserBatch.saveDistrictCalcCommand(c, d.region, d.name, d.estate_type2, lastDay, "fangjia-priceanalysis-0")
+ buffer << cmd
+ if(buffer.size() == 1000){
+ COMMAND_COLL.insert(buffer)
+ buffer.clear()
+ }
+ println "[add] "+c+" "+ d.name+" "+DateUtil.formatByPattern(lastDay, "yyyy-MM-dd")
+ }
+ }
+ }
+ }
+
+ if(!buffer.isEmpty()){
+ COMMAND_COLL.insert(buffer)
+ }
+
+ }
+ }
}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy
index 2d7515aa7..0fbf352f7 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy
@@ -7,6 +7,7 @@ import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.CommonUtils
+import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
@@ -34,7 +35,7 @@ class ThirdPriceMappingBatch extends BaseBatch{
final DBCollection dataProcessColl = MongoUtils.getCollectionByName("fangjia_craw", "data_process", "192.168.0.235", 27018)
- def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
+ def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
static main(args) {
runBatch(new ThirdPriceMappingBatch(), args);
@@ -76,7 +77,17 @@ class ThirdPriceMappingBatch extends BaseBatch{
address = address + nmd.district
}
address = address + nmd.name
- def jsonObject = FangjiaClientUtils.newMatch(nmd.city, address, "property", true)
+ def jsonObject = FangjiaClientUtils.newMatch(nmd.city, address, "property", false)
+ if(!jsonObject){
+ if(DataUtil.isPreciseAddress(nmd.address)){
+ address = nmd.city
+ if(nmd.district){
+ address = address + nmd.district
+ }
+ address = address + nmd.address
+ jsonObject = FangjiaClientUtils.newMatch(nmd.city, address, "property", false)
+ }
+ }
if(jsonObject) {
def udata = new BasicDBObject()
udata.put("match_id",jsonObject.get("id"))
@@ -161,7 +172,7 @@ class ThirdPriceMappingBatch extends BaseBatch{
}
def addOldMatchResult(){
- def citys = ["ÉϺ£"]//CommonUtils.findAllCity()
+ def citys = CommonUtils.findAllCity()
def keys = new BasicDBObject()
keys.put("_id", 1)
@@ -199,7 +210,7 @@ class ThirdPriceMappingBatch extends BaseBatch{
println addCount.get()+"/"+total +" [¾ÉÆ¥ÅäÕÒµ½] "
}else{
- println addCount.get()+"/"+total +" [¾ÉÆ¥ÅäûÕÒµ½] "
+ println addCount.get()+"/"+total +" [¾ÉÆ¥ÅäûÕÒµ½] "+d._id.toString()
}
} catch (Exception e) {
e.printStackTrace()
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy
index f1df981ef..1d63a790f 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy
@@ -22,6 +22,7 @@ import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.QueryOperators
import net.sf.json.JSONObject
+import org.apache.commons.lang3.SerializationUtils
import org.apache.commons.lang3.StringUtils
import org.apache.tools.ant.util.DateUtils
import org.bson.types.ObjectId
@@ -61,7 +62,7 @@ class DataStandardBatch extends BaseBatch{
private static def districtMap = [:];
private static def limit = 5000;
- def static final SHORT_DATE = Integer.parseInt(com.fangjia.script.util.DateUtils.addMonth(com.fangjia.script.util.DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
+ def static final SHORT_DATE = Integer.parseInt(com.fangjia.script.util.DateUtils.addMonth(com.fangjia.script.util.DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
def static final statusMap = ["³ÇÊÐδƥÅä":1,"ÇøÓòδƥÅä":2,"Ð¡ÇøÆ¥Åäʧ°Ü":3,"Ð¡ÇøÆ¥Åä³É¹¦":4,"Òì³£´íÎó":5, "ÎÞ¼Û¸ñ":6, "ÎÞÃû³Æ":7]
def static final websites = ["·¿¹À¹À","Öî¸ðÕÒ·¿","°²¾Ó¿Í","ËÑ·¿","ÖÐԵزú","¸Ï¼¯", "Á´¼Ò","Íò©Ê¿","³ÇÊз¿²ú","±´¿ÇÕÒ·¿","Q·¿","Q·¿(ÐÂ)","ÖÐԵزú(ÐÂ)","ÎÒ°®ÎÒ¼Ò","ÂóÌï","·¿Í¾","C21","ÀÖ¾Ó","·¿²ú³¬ÊÐ","58ͬ³Ç","365ÌÔ·¿","¼ªÎÝ","ÐÒ¸£Àï","Öйú·¿¼ÛÐÐÇé","³£ÖÝס·¿"]
@@ -92,7 +93,7 @@ class DataStandardBatch extends BaseBatch{
cityMap.each{city,alias->
println "¿ªÊ¼¼ÓÔØ¡¾${city}¡¿"
-// if(city == "¸£ÖÝ"){
+// if(city == "ÉϺ£"){
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"])
districtMap.putAll(detailMap)
// }
@@ -463,7 +464,7 @@ class DataStandardBatch extends BaseBatch{
// }catch (Exception e){
// addr = data.address;
// }
- def copyObj = data
+ def copyObj = SerializationUtils.clone(data)
// copyObj.address = addr
district = getMatchAddressFromMap(copyObj)
if(district){
@@ -492,7 +493,7 @@ class DataStandardBatch extends BaseBatch{
}
if(!district){
- def copyObj = data
+ def copyObj = SerializationUtils.clone(data)
copyObj.address = addr
district = getOpenApiMatch(copyObj)
if(!district && addr){
@@ -906,7 +907,13 @@ class DataStandardBatch extends BaseBatch{
def getOpenApiMatch(def data){
if(StringUtils.isNotEmpty(data.name)){
- def jsonObject = OpenApiClient.addressMatch(data.city, data.name, "property", false, data.region, null, "fuzzy", false)
+ def jsonObject = null
+ if(DataUtil.isPreciseAddress(data.address)){
+ jsonObject = OpenApiClient.addressMatch(data.city, data.address, "property", false, data.region, data.name, "fuzzy", false)
+ }else{
+ jsonObject = OpenApiClient.addressMatch(data.city, data.name, "property", false, data.region, null, "fuzzy", false)
+ }
+
if(jsonObject && jsonObject.credit>0 ){
def dbo = [city:jsonObject.city, region:jsonObject.district, name:jsonObject.name, address:jsonObject.address,estate_type:jsonObject.propertyType, credit:jsonObject.credit]
if(dbo){
@@ -923,7 +930,7 @@ class DataStandardBatch extends BaseBatch{
def findAllNoMatchData(){
DBObject qdata = new BasicDBObject();
qdata.put("s_date", new BasicDBObject('$gte', SHORT_DATE))
-// qdata.put("city","¸£ÖÝ")
+// qdata.put("city","ÉϺ£")
// qdata.put("source","°²¾Ó¿Í")
// qdata.put("name",Pattern.compile("¹¨¼ÒÕ¬.*"))
qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false));