From 3b3615b2bf4f5e757d2f0e2fcfea4f6af0bc19ff Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 20 Jun 2024 21:43:11 +0800 Subject: [PATCH] update --- .../script/ccbevaluation/AddSzDealData.groovy | 18 +++++++------- .../ccbevaluation/BigdataDealBatch.groovy | 24 ++++++++++++------- .../ccbevaluation/CcbCommonUtils.groovy | 3 ++- .../ccbevaluation/GuiDingPriceBatch.groovy | 2 +- .../script/ccbevaluation/TradeStatLog.groovy | 8 +++---- .../script/commData/CommonDatautils.groovy | 2 +- .../com/fangjia/script/gj/OpenApiUtils.groovy | 5 +++- .../district/SeaweedDataMatchProcess.groovy | 12 ++++------ 8 files changed, 42 insertions(+), 32 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy index cb4abf8cf..ff77ef91f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy @@ -9,7 +9,7 @@ import com.mongodb.DBCollection /** * 深圳公众号成交数据录入 */ -def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳2024年3月份二手房成交数据汇总.xls", 0, 1) +def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\20240619--深圳成交数据.xlsx", 0, 1) def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) @@ -72,19 +72,21 @@ for(def d : datas){ } - map.area = d['F'].toString().toDouble() - map.total_price = (int)(d['G'].toString().toDouble().toInteger()*10000) - map.avg_price = (int)(d['H'].toString().toDouble()) + map.area = d['F'].replace("平","").toString().toDouble() + map.total_price = (int)(d['G'].replace("万","").toString().toDouble().toInteger()*10000) + map.avg_price = (int)(d['H'].replace("元/平","").toString().toDouble()) if(d['I']){ - map.sell_price = (int)(d['I'].toString().toDouble()*10000) + map.sell_price = (int)(d['I'].replace("万","").toString().toDouble()*10000) } + map.trade_date = d['J'] // def trade_date = d['J'] // if(trade_date){ // map.trade_date = DateUtils.parseFromTightByPattern(trade_date,'yyyy/MM/dd') -// }else{ - def trade_date1 = DateUtils.getRandomDate("2024-03-01", "2024-03-31") - map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd") +// } +// else{ +// def trade_date1 = DateUtils.getRandomDate("2024-03-01", "2024-03-31") +// map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd") // } map.source = "深房家族" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 2955ed1e6..8bbea25d3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -35,7 +35,7 @@ import java.util.regex.Pattern @BatchOwner(Watcher.GaoJuan) class BigdataDealBatch extends BaseBatch{ - def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -24, "yyyyMM")); + def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd")); @@ -726,18 +726,24 @@ class BigdataDealBatch extends BaseBatch{ t.name = t.name +"商住楼" } } - rMap = SeaweedDataMatchProcess.doMatch(t.city, region, t.name, null); - if(!rMap){ - - } - if(!rMap && t.address){ - def flag = t.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街|巷).+[弄号]$') + def flag = false + def address = null + if(t.address){ + flag = t.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街|巷).+[弄号]$') if(flag){ - rMap = SeaweedDataMatchProcess.doMatch(t.city, region, null, t.address); + address = t.address } } + + rMap = SeaweedDataMatchProcess.doMatch(t.city, region, t.name, address); + if(!rMap && !t.name.toString().endsWith("小区")){ + rMap = SeaweedDataMatchProcess.doMatch(t.city, region, t.name+"小区", address); + } + if(rMap && rMap.region==region){ CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id) + }else if(rMap && rMap.address && address && rMap.region!=region && address.contains(rMap.address)){ + CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id) }else { if (rMap) { println "区域不同" @@ -755,7 +761,7 @@ class BigdataDealBatch extends BaseBatch{ } } - ); + ) } latch.await() println city +" "+trades.size()+" "+new Date().toString() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 7a4062270..7dac52e9c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -634,7 +634,8 @@ class CcbCommonUtils { q.put("city", city) q.put("trade_date", new BasicDBObject('$gte', startDate)) q.put("match_id", new BasicDBObject('$exists', false)) -// q.put("district_name","中海寰宇天下") +// q.put("district_name","新明胡同") +// q.put("address","桂苑大道135号") def dbcursor = DEAL_PRICE_COLL.find(q) def total = dbcursor.count() def rets = new HashSet() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index c43a791b6..5c735e4c3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2024-05-24", "2024-06-08") + def prices = CcbCommonUtils.findPersonPrice(city, "2024-05-24", "2024-06-21") // def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") if(prices){ def distMap = getDistrictMap(city) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index 7ab2fc7e4..57bdfdbd9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -40,8 +40,8 @@ class TradeStatLog { static main(args) { def task = new TradeStatLog() - task.run()//统计最近3个月抓取的成交数据量 -// task.exportTrade()//导出未入库成交 +// task.run()//统计最近3个月抓取的成交数据量 + task.exportTrade()//导出未入库成交 // task.thirdPartyStat() // println Calendar.getInstance().get(Calendar.YEAR) // task.averageStat() @@ -499,8 +499,8 @@ class TradeStatLog { def headers = ["ID", "城市", "区域", "名称", "来源","价格", "面积","地址", "成交时间","过滤原因"] def rets = [] rets << headers - def startDate = "2023-08-01" - def endDate = "2023-10-01" + def startDate = "2024-04-01" + def endDate = "2024-06-01" for(def city : citys){ def trades = findTradesDetail(city, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd")) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index e9fa92613..3bdde3598 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -4679,7 +4679,7 @@ class CommonDatautils { def q = new BasicDBObject() q.put("type","乡镇街道") - q.put("province",new BasicDBObject('$in', ['福建省','重庆市'])) + q.put("province",new BasicDBObject('$in', ['浙江省','江苏省'])) def keys = new BasicDBObject() keys.put("province",1) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy index cbd2845ec..6d931d1cc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy @@ -13,12 +13,15 @@ import net.sf.json.groovy.JsonSlurper import org.apache.commons.lang.StringUtils class OpenApiUtils { - static def matchDistrict(def city, def address, def suggestDistrict, def preciseLevelflag){ + static def matchDistrict(def city, def address, def suggestDistrict, def preciseLevelflag, def suggestName){ address = address.replace("(","(").replace(")",")").replace(" ","").replace("#","").replace("<","").replace(">","").replace("%","").replace("\"","") def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&suggestDistrict="+suggestDistrict+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1" if(preciseLevelflag){ url = url +"&preciseLevel=strict" } + if(suggestName){ + url = url +"&suggestName="+suggestName + } String content = new HttpRequest().get(url, "utf-8"); def rMap = [:] if(content){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy index c26ded24c..ba72b03de 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy @@ -36,18 +36,16 @@ class SeaweedDataMatchProcess { } } - if(!rMap && name){ - rMap = OpenApiUtils.matchDistrict(city, name, region, false) - } - - - if(!rMap && address){ + if(!rMap && address && name){ def flag = address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$') if(flag){ - rMap = OpenApiUtils.matchDistrict(city, address, region, true) + rMap = OpenApiUtils.matchDistrict(city, address, region, true, name) } } + if(!rMap && name){ + rMap = OpenApiUtils.matchDistrict(city, name, region, false, null) + } // if(rMap && rMap.id && region.contains(rMap.region)) { // def obj = CommonDatautils.findDistrictById(rMap.id.toString()); // if (obj) {