From 7cfcc209ece8d9ca5146733514c24b322ac58974 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 8 Oct 2024 18:08:59 +0800 Subject: [PATCH 1/3] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=9C=B0=E5=9D=80?= =?UTF-8?q?=E5=8C=B9=E9=85=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../data_handle/ThirdPriceMappingBatch.groovy | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy index 2d7515aa7..0fbf352f7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy @@ -7,6 +7,7 @@ import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.gj.district.FangjiaClientUtils import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DataUtil import com.fangjia.script.util.DateUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject @@ -34,7 +35,7 @@ class ThirdPriceMappingBatch extends BaseBatch{ final DBCollection dataProcessColl = MongoUtils.getCollectionByName("fangjia_craw", "data_process", "192.168.0.235", 27018) - def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); static main(args) { runBatch(new ThirdPriceMappingBatch(), args); @@ -76,7 +77,17 @@ class ThirdPriceMappingBatch extends BaseBatch{ address = address + nmd.district } address = address + nmd.name - def jsonObject = FangjiaClientUtils.newMatch(nmd.city, address, "property", true) + def jsonObject = FangjiaClientUtils.newMatch(nmd.city, address, "property", false) + if(!jsonObject){ + if(DataUtil.isPreciseAddress(nmd.address)){ + address = nmd.city + if(nmd.district){ + address = address + nmd.district + } + address = address + nmd.address + jsonObject = FangjiaClientUtils.newMatch(nmd.city, address, "property", false) + } + } if(jsonObject) { def udata = new BasicDBObject() udata.put("match_id",jsonObject.get("id")) @@ -161,7 +172,7 @@ class ThirdPriceMappingBatch extends BaseBatch{ } def addOldMatchResult(){ - def citys = ["上海"]//CommonUtils.findAllCity() + def citys = CommonUtils.findAllCity() def keys = new BasicDBObject() keys.put("_id", 1) @@ -199,7 +210,7 @@ class ThirdPriceMappingBatch extends BaseBatch{ println addCount.get()+"/"+total +" [旧匹配找到] " }else{ - println addCount.get()+"/"+total +" [旧匹配没找到] " + println addCount.get()+"/"+total +" [旧匹配没找到] "+d._id.toString() } } catch (Exception e) { e.printStackTrace() From 8167a69773fc60a0c3b58590156271413110d5a4 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Wed, 9 Oct 2024 13:59:44 +0800 Subject: [PATCH 2/3] =?UTF-8?q?=E7=AC=AC=E4=B8=89=E6=96=B9=E5=8C=B9?= =?UTF-8?q?=E9=85=8D=E4=B8=8A=E4=B8=AA=E6=9C=88=E8=A1=A5=E8=B7=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/commData/CommonDatautils.groovy | 49 ++++++++++++- .../gj/data_handle/AddCityRegionPrice.groovy | 68 ++++++++++++++++++- .../thirdPriceCrawl/DataStandardBatch.groovy | 19 ++++-- 3 files changed, 126 insertions(+), 10 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index f710a70fe..94668a4df 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -3475,6 +3475,36 @@ class CommonDatautils { } return obj; } + + + + + static def queryFirstOrLastRentPrice(city, region, name, sort){ + DBObject queryCondition = new BasicDBObject() + queryCondition.put("city", city); + if(region){ + queryCondition.put("region", region); + } + + if(name){ + queryCondition.put("cat", "district"); + queryCondition.put("district_name", name) + }else{ + if(region){ + queryCondition.put("cat", "region") + }else{ + queryCondition.put("cat", "city") + } + } + + DBCursor cur = SQMETRE_RENT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", sort)).limit(1) + cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT) + def obj = 0; + while(cur.hasNext()) { + obj = cur.next() + } + return obj; + } static def queryCityRegionPrice(city, region, startDate, endDate){ DBObject queryCondition = new BasicDBObject() @@ -3691,11 +3721,26 @@ class CommonDatautils { return count; } + static def findSqmetreRent(def city, def region, def name, def weekend){ def q = new BasicDBObject() q.put("city", city) - q.put("region", region) - q.put("district_name", name) + if(region){ + q.put("region", region) + } + if(name){ + q.put("district_name", name) + q.put("cat", "district") + }else{ + if(!region){ + if(city){ + q.put("cat", "city") + }else{ + q.put("cat", "region") + } + } + } + q.put("weekend", weekend) def dbcur = SQMETRE_RENT_COLL.find(q) def dbo = null diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy index 667f06d6d..cc42938d7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddCityRegionPrice.groovy @@ -2,20 +2,30 @@ package com.fangjia.script.gj.data_handle import com.fangjia.core.util.DateUtil import com.fangjia.script.averager.InsertMonitorCalcCommandBatch +import com.fangjia.script.averager.InsertMonitorSquareRentCalcCommandBatch import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.mongodb.DBCollection +import com.mongodb.MongoClient class AddCityRegionPrice { + + public final static DBCollection COMMAND_COLL = new MongoClient("log.mongo.db.fangjia.com", 52002).getDB('fangjia_log').getCollection('calc_command') + static main(args) { def task = new AddCityRegionPrice() // task.addRegionPrice() // task.addCityprice() - task.addDistrictprice() +// task.addDistrictprice() +// task.addCityRentPrice() + task.addDistrictRentprice() + } def addRegionPrice(){ InsertMonitorCalcCommandBatch insertBatch = new InsertMonitorCalcCommandBatch() - def citys = CommonDatautils.findAllCity() + def citys = ["文山州"]//CommonDatautils.findAllCity() def add = 0 for(def c : citys){ def regions = CommonDatautils.getCityRegions(c) @@ -94,4 +104,58 @@ class AddCityRegionPrice { } } + + def addCityRentPrice(){ + InsertMonitorSquareRentCalcCommandBatch inserBatch = new InsertMonitorSquareRentCalcCommandBatch(); + def citys = CommonDatautils.findAllCity() + for(def c : citys){ + def startObj = CommonDatautils.queryFirstOrLastRentPrice(c, null, null, 1) + def endObj = CommonDatautils.queryFirstOrLastRentPrice(c, null, null, -1) + if(startObj && endObj){ + def months = DateUtils.getBetweenMonth(DateUtil.formatByPattern(startObj.weekend, "yyyyMM"),"202409") + months.each {m-> + def lastDay = DateUtils.truncateDay(DateUtils.getLastDayOfMonth(DateUtil.parseDate(m+"01", "yyyyMMdd"))) + def obj = CommonDatautils.findSqmetreRent(c, null, null, lastDay) + if(!obj){ + inserBatch.saveCityCalcCommand(c, null, null, lastDay, "fangjia-priceanalysis-0") + println "[add] "+c+" "+DateUtil.formatByPattern(lastDay, "yyyy-MM-dd") + } + } + } + } + } + + def addDistrictRentprice(){ + InsertMonitorSquareRentCalcCommandBatch inserBatch = new InsertMonitorSquareRentCalcCommandBatch(); + def citys = CommonDatautils.findAllCity() + for(def c : citys){ + def buffer = [] + def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "estate_type2"]) + for(def d : districts){ + def startObj = CommonDatautils.queryFirstOrLastRentPrice(c, d.region, d.name, 1) +// def endObj = CommonDatautils.queryFirstOrLastRentPrice(c, d.region, d.name, -1) + if(startObj){ + def months = DateUtils.getBetweenMonth(DateUtil.formatByPattern(startObj.weekend, "yyyyMM"),"202409") + months.each {m-> + def lastDay = DateUtils.truncateDay(DateUtils.getLastDayOfMonth(DateUtil.parseDate(m+"01", "yyyyMMdd"))) + def obj = CommonDatautils.findSqmetreRent(c, d.region, d.name, lastDay) + if(!obj){ + def cmd = inserBatch.saveDistrictCalcCommand(c, d.region, d.name, d.estate_type2, lastDay, "fangjia-priceanalysis-0") + buffer << cmd + if(buffer.size() == 1000){ + COMMAND_COLL.insert(buffer) + buffer.clear() + } + println "[add] "+c+" "+ d.name+" "+DateUtil.formatByPattern(lastDay, "yyyy-MM-dd") + } + } + } + } + + if(!buffer.isEmpty()){ + COMMAND_COLL.insert(buffer) + } + + } + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy index f1df981ef..1d63a790f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy @@ -22,6 +22,7 @@ import com.mongodb.DBCollection import com.mongodb.DBObject import com.mongodb.QueryOperators import net.sf.json.JSONObject +import org.apache.commons.lang3.SerializationUtils import org.apache.commons.lang3.StringUtils import org.apache.tools.ant.util.DateUtils import org.bson.types.ObjectId @@ -61,7 +62,7 @@ class DataStandardBatch extends BaseBatch{ private static def districtMap = [:]; private static def limit = 5000; - def static final SHORT_DATE = Integer.parseInt(com.fangjia.script.util.DateUtils.addMonth(com.fangjia.script.util.DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def static final SHORT_DATE = Integer.parseInt(com.fangjia.script.util.DateUtils.addMonth(com.fangjia.script.util.DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6, "无名称":7] def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋","幸福里","中国房价行情","常州住房"] @@ -92,7 +93,7 @@ class DataStandardBatch extends BaseBatch{ cityMap.each{city,alias-> println "开始加载【${city}】" -// if(city == "福州"){ +// if(city == "上海"){ def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"]) districtMap.putAll(detailMap) // } @@ -463,7 +464,7 @@ class DataStandardBatch extends BaseBatch{ // }catch (Exception e){ // addr = data.address; // } - def copyObj = data + def copyObj = SerializationUtils.clone(data) // copyObj.address = addr district = getMatchAddressFromMap(copyObj) if(district){ @@ -492,7 +493,7 @@ class DataStandardBatch extends BaseBatch{ } if(!district){ - def copyObj = data + def copyObj = SerializationUtils.clone(data) copyObj.address = addr district = getOpenApiMatch(copyObj) if(!district && addr){ @@ -906,7 +907,13 @@ class DataStandardBatch extends BaseBatch{ def getOpenApiMatch(def data){ if(StringUtils.isNotEmpty(data.name)){ - def jsonObject = OpenApiClient.addressMatch(data.city, data.name, "property", false, data.region, null, "fuzzy", false) + def jsonObject = null + if(DataUtil.isPreciseAddress(data.address)){ + jsonObject = OpenApiClient.addressMatch(data.city, data.address, "property", false, data.region, data.name, "fuzzy", false) + }else{ + jsonObject = OpenApiClient.addressMatch(data.city, data.name, "property", false, data.region, null, "fuzzy", false) + } + if(jsonObject && jsonObject.credit>0 ){ def dbo = [city:jsonObject.city, region:jsonObject.district, name:jsonObject.name, address:jsonObject.address,estate_type:jsonObject.propertyType, credit:jsonObject.credit] if(dbo){ @@ -923,7 +930,7 @@ class DataStandardBatch extends BaseBatch{ def findAllNoMatchData(){ DBObject qdata = new BasicDBObject(); qdata.put("s_date", new BasicDBObject('$gte', SHORT_DATE)) -// qdata.put("city","福州") +// qdata.put("city","上海") // qdata.put("source","安居客") // qdata.put("name",Pattern.compile("龚家宅.*")) qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false)); From 063e388af3bf524da4141da25ac64b2d28da960c Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Wed, 9 Oct 2024 14:15:12 +0800 Subject: [PATCH 3/3] =?UTF-8?q?=E5=8E=BB=E6=8E=89=E6=88=BF=E4=BA=A7?= =?UTF-8?q?=E5=A4=A7=E6=95=B0=E6=8D=AE=E4=B8=AD2=E4=B8=AA=E9=A1=B5?= =?UTF-8?q?=E9=9D=A2=E7=9A=84banner?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/webapp/evaluation/reportNew.ftl | 24 +++++++++---------- .../src/main/webapp/trend/trend.ftl | 24 +++++++++---------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/projects/fangjia-coresite/src/main/webapp/evaluation/reportNew.ftl b/projects/fangjia-coresite/src/main/webapp/evaluation/reportNew.ftl index 4be5992ee..d93369690 100644 --- a/projects/fangjia-coresite/src/main/webapp/evaluation/reportNew.ftl +++ b/projects/fangjia-coresite/src/main/webapp/evaluation/reportNew.ftl @@ -72,18 +72,18 @@ <#include "/header/v6/_header.ftl"/> <#include "/evaluation/checkUserNewRedirct.ftl"/> -
-
-
-
-

- -
-
-
-
+<#--
--> +<#--
--> +<#--
--> +<#--
--> +<#--

--> +<#--
--> +<#--
鐢宠璇曠敤
--> +<#--
--> +<#--
--> +<#--
--> +<#--
--> +<#--
-->
diff --git a/projects/fangjia-coresite/src/main/webapp/trend/trend.ftl b/projects/fangjia-coresite/src/main/webapp/trend/trend.ftl index 57168fb3d..33108e5f4 100644 --- a/projects/fangjia-coresite/src/main/webapp/trend/trend.ftl +++ b/projects/fangjia-coresite/src/main/webapp/trend/trend.ftl @@ -52,18 +52,18 @@
${affectValue}
-
-
-
-
-

- -
-
-
-
+<#--
--> +<#--
--> +<#--
--> +<#--
--> +<#--

--> +<#--
--> +<#--
鐢宠璇曠敤
--> +<#--
--> +<#--
--> +<#--
--> +<#--
--> +<#--
-->