From 4b3dee53081d2257c7535ff0664964d9e203f2f0 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 10 Sep 2021 11:01:36 +0800 Subject: [PATCH] update --- .../ccbevaluation/CcbCommonUtils.groovy | 13 ++ .../DistrictGuidePriceBatch.groovy | 85 ++++++++++++ .../ccbevaluation/GuiDingPriceBatch.groovy | 26 ++-- .../script/ccbevaluation/LoadGuide.groovy | 98 ++++++++++++++ .../fangjia/script/ccbevaluation/Test.groovy | 104 +++++++-------- .../ccbevaluation/ThirdPartyPriceBatch.groovy | 59 +++++++-- .../fangjia/script/gj/YouDaDataParser.groovy | 16 +-- .../fangjia/script/gj/data/FieldStat.groovy | 37 +++++- .../guangfa/GuangfaBlankSelectDistrict.groovy | 122 +++++++++++++++++- .../thirdPriceCrawl/SynchDistrictPrice.groovy | 20 +-- .../TestCrawlTotalCount.groovy | 4 +- 11 files changed, 485 insertions(+), 99 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LoadGuide.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 9a41d91ad..999d4b347 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -27,6 +27,8 @@ class CcbCommonUtils { public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com") + public static final DBCollection DISTRICT_GUIDE_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "district_guide_price", "mongo.db.fangjia.com", 28017) + public static def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42"] public static DSCollection dataStoreConnection = null; @@ -75,6 +77,17 @@ class CcbCommonUtils { return rets } + public static def findGuidePrice(city){ + def q = new BasicDBObject() + q.put("city", city) + def dbcursor = DISTRICT_GUIDE_PRICE_COLL.find(q) + def rets = [] + for(def d : dbcursor){ + rets << d + } + return rets + } + public static def findTradesDetail(dist, startDate, endDate, startPrice, endPrice){ def q = new BasicDBObject() q.put("city", dist.city) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy new file mode 100644 index 000000000..3f73a32d6 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy @@ -0,0 +1,85 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils + + +@BatchOwner(Watcher.GaoJuan) +class DistrictGuidePriceBatch extends BaseBatch{ + + + static main(args) { + runBatch(new DistrictGuidePriceBatch(), args); + } + + @Override + public void process(Object args) { + def citys = ["深圳"]//CcbCommonUtils.citys; + def rows = [] + + for (def city : citys) { + def cityCodeDatas = CcbCommonUtils.findCityCodeValue() + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + + //添加自己库中的人工指导价 + def prices = CcbCommonUtils.findGuidePrice(city) + if(prices){ + def distMap = getDistrictMap(city) + def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices) + rows.addAll(rets) + println city + " " + rets.size() + } + } + +// def headers = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导价格","类型","备注","创建时间","更新时间","来源公司"] +// FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\district_guied_price.dat", headers, "utf-8", false); + + } + + + def getDistrictMap(def city){ + def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]); + def distMap = [:] + for(def d : districts){ + distMap.put(d.region+"##"+d.name, d) + } + return distMap; + } + + def createCityDatas(cityCodeDatas, distCodeMap, distMap, prices){ + def rets = [] + if(prices){ + for(def ps : prices){ + def dist = distMap.get(ps.match_region+"##"+ps.match_name) + if(dist){ + def ret = [] + def areaData = distCodeMap.get(dist.region) + if(areaData){ + ret << ps._id.toString() + ret << dist._id.toString() + ret << dist.name + ret << cityCodeDatas.get(dist.city) + ret << dist.city + ret << areaData.district_code + ret << areaData.district + ret << ps.avg_price + ret << ps.source_type + ret << "" + ret << DateUtils.formatDateByPattern(ps.c_date, "yyyy-MM-dd HH:mm:ss") + ret << DateUtils.formatDateByPattern(ps.c_date, "yyyy-MM-dd HH:mm:ss") + ret << "FJW" + rets << ret + } + }else{ + println ps.match_region+"##"+ps.match_name + } + } + } + return rets; + } + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 1842c3e5b..68eb41ed3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -53,25 +53,25 @@ class GuiDingPriceBatch extends BaseBatch{ } //添加搜房的参考价 -// def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]); + def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]); // def datas = findFangDatas(districts) // def fangRets = createDatas(cityCodeDatas, distCodeMap, datas) // rows.addAll(fangRets) //添加保亭文昌的指导价 -// def cityDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\文昌-保亭待调查小区价格 - 210902(1).xlsx") -// def cityDataMap = [:] -// for(def d : cityDatas){ -// cityDataMap.put(d['A'], d['E']) -// } -// def personDatas = findPersonDatas(districts, cityDataMap); -// def personRets = createDatas(cityCodeDatas, distCodeMap, personDatas) -// rows.addAll(personRets) + def cityDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\文昌-保亭待调查小区价格 - 210902(1).xlsx") + def cityDataMap = [:] + for(def d : cityDatas){ + cityDataMap.put(d['A'], d['E']) + } + def personDatas = findPersonDatas(districts, cityDataMap); + def personRets = createDatas(cityCodeDatas, distCodeMap, personDatas) + rows.addAll(personRets) } - def headers = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"] - FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\guiding_price.dat", headers, "utf-8", true); + def headers = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"] + FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\guiding_price.dat", headers, "utf-8", false); // def rets = [] // rets << headers @@ -146,8 +146,8 @@ class GuiDingPriceBatch extends BaseBatch{ ret << ps._id.toString() ret << dist._id.toString() ret << dist.name - ret << cityCodeDatas.city_code - ret << cityCodeDatas.fangjia_city + ret << cityCodeDatas.get(dist.city) + ret << dist.city ret << areaData.district_code ret << areaData.district ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LoadGuide.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LoadGuide.groovy new file mode 100644 index 000000000..c6fa7f82c --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LoadGuide.groovy @@ -0,0 +1,98 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +class LoadGuide { + public static final DBCollection DISTRICT_GUIDE_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "district_guide_price", "mongo.db.fangjia.com", 28017) + + static main(args) { + def task = new LoadGuide() + task.run() + } + + def run(){ +// saveDongGuangPrice("D:\\数据\\建行\\房产估值\\指导价\\东莞二手房政府指导价.xls") +// saveChengduoPrice("D:\\数据\\建行\\房产估值\\指导价\\成都部分住宅小区二手房成交参考价格201-286-258.xlsx") +// savePrice("D:\\数据\\建行\\房产估值\\指导价\\绍兴指导价小区_0909.xlsx") +// savePrice("D:\\数据\\建行\\房产估值\\指导价\\宁波交易参考价格.xlsx") +// savePrice("D:\\数据\\建行\\房产估值\\指导价\\西安成交参考价格.xlsx") +// savePrice("D:\\数据\\建行\\房产估值\\指导价\\无锡100 广州96 二手住房成交参考价格表.xlsx") + savePrice("D:\\数据\\建行\\房产估值\\指导价\\深圳指导价 - 210909 -.xlsx") + + } + + def savePrice(filePath){ + def datas = ExcelUtils.read(filePath,0, 1) + def rets = [] + for(def d : datas){ + if(d['E']){ + def o = new BasicDBObject() + o.city = d['E'] + o.region = d['A'] + o.name = d['C'] + o.avg_price = d['D'].toString().toDouble().toInteger() + o.match_city = d['E'] + o.match_region = d['F'] + o.match_name = d['G'] + o.source_type = 1 //1:成交参考价 2:参考价 + o.c_date = new Date() + rets << o + } + } + if(rets){ + println rets.size() + DISTRICT_GUIDE_PRICE_COLL.insert(rets) + } + } + + def saveDongGuangPrice(def filePath){ + def datas = ExcelUtils.read(filePath,0, 0) + def rets = [] + for(def d : datas){ + if(d['E']){ + def o = new BasicDBObject() + o.city = d['E'] + o.region = d['A'] + o.name = d['B'] + o.avg_price = d['D'].toString().toDouble().toInteger() + o.match_city = d['E'] + o.match_region = d['F'] + o.match_name = d['G'] + o.house_count = d['C'].toString().toDouble().toInteger() + o.source_type = 1 //1:成交参考价 2:参考价 + o.c_date = new Date() + rets << o + } + } + if(rets){ + DISTRICT_GUIDE_PRICE_COLL.insert(rets) + } + } + + def saveChengduoPrice(def filePath){ + def datas = ExcelUtils.read(filePath,2, 0) + def rets = [] + for(def d : datas){ + if(d['E']){ + def o = new BasicDBObject() + o.city = d['E'] + o.region = d['B'] + o.name = d['C'] + o.avg_price = d['D'].toString().toDouble().toInteger() + o.match_city = d['E'] + o.match_region = d['F'] + o.match_name = d['G'] + o.source_type = 1 //1:成交参考价 2:参考价 + o.c_date = new Date() + rets << o + } + } + if(rets){ + DISTRICT_GUIDE_PRICE_COLL.insert(rets) + } + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index d7b05ed1b..18dc9f380 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -59,7 +59,7 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw -def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0807 - 副本.xlsx" +def fileName = "d:\\数据\\建行\\房产估值\\基础数据交付情况_0903.xlsx" //def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\小区字段.xlsx",0,1) //def dealMap = [:] @@ -83,58 +83,58 @@ def cityCodeDatas = CcbCommonUtils.findCityCodeValue() // if (city) { // def cityid = cityCodeDatas.get(city) // def total = findCount(cityid, sqlInstance) -// d['K'] = total +// d['M'] = total // } //},0) -//小区字段覆盖率 -ExcelUtils.multiThreadingModify(fileName,{d-> - def city = d['A'] - if(city){ - def cityid = cityCodeDatas.get(city) - def total = findCount(cityid, sqlInstance) - d['AA'] = total - - def a = findEstateType(cityid, sqlInstance) - d['AB'] = DataHandleUtils.getDoubleRound(2, a*100/total) - - def b = findFiledCount(cityid, 'buildyear', sqlInstance) - d['AC'] = DataHandleUtils.getDoubleRound(2, b*100/total) - - def c = findFiledCount(cityid, 'address', sqlInstance) - d['AD'] = DataHandleUtils.getDoubleRound(2, c*100/total) - - def d1 = findFiledCount(cityid, 'ZTS', sqlInstance) - d['AE'] = DataHandleUtils.getDoubleRound(2, d1*100/total) - - def e = findFiledCount(cityid, 'landarea', sqlInstance) - d['AF'] = DataHandleUtils.getDoubleRound(2, e*100/total) - - def f = findFiledCount(cityid, 'buildarea', sqlInstance) - d['AG'] = DataHandleUtils.getDoubleRound(2, f*100/total) - - def g = findFiledCount(cityid, 'plotratio', sqlInstance) - d['AH'] = DataHandleUtils.getDoubleRound(2, g*100/total) - - def h = findFiledCount(cityid, 'greeningrate', sqlInstance) - d['AI'] = DataHandleUtils.getDoubleRound(2, h*100/total) - - def i = findFiledCount(cityid, 'latitude', sqlInstance) - d['AJ'] = DataHandleUtils.getDoubleRound(2, i*100/total) - - def j = findFiledCount(cityid, 'parkingcnt', sqlInstance) - d['AK'] = DataHandleUtils.getDoubleRound(2, j*100/total) - println d - } -},1) - -//def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0807-1.xlsx" -//def dealMap = findDealCount(sqlInstance) +////小区字段覆盖率 //ExcelUtils.multiThreadingModify(fileName,{d-> -// if(d['C']){ -// d['Q'] = dealMap.get(d['C']) +// def city = d['A'] +// if(city){ +// def cityid = cityCodeDatas.get(city) +// def total = findCount(cityid, sqlInstance) +// d['AA'] = total +// +// def a = findEstateType(cityid, sqlInstance) +// d['AB'] = DataHandleUtils.getDoubleRound(2, a*100/total) +// +// def b = findFiledCount(cityid, 'buildyear', sqlInstance) +// d['AC'] = DataHandleUtils.getDoubleRound(2, b*100/total) +// +// def c = findFiledCount(cityid, 'address', sqlInstance) +// d['AD'] = DataHandleUtils.getDoubleRound(2, c*100/total) +// +// def d1 = findFiledCount(cityid, 'ZTS', sqlInstance) +// d['AE'] = DataHandleUtils.getDoubleRound(2, d1*100/total) +// +// def e = findFiledCount(cityid, 'landarea', sqlInstance) +// d['AF'] = DataHandleUtils.getDoubleRound(2, e*100/total) +// +// def f = findFiledCount(cityid, 'buildarea', sqlInstance) +// d['AG'] = DataHandleUtils.getDoubleRound(2, f*100/total) +// +// def g = findFiledCount(cityid, 'plotratio', sqlInstance) +// d['AH'] = DataHandleUtils.getDoubleRound(2, g*100/total) +// +// def h = findFiledCount(cityid, 'greeningrate', sqlInstance) +// d['AI'] = DataHandleUtils.getDoubleRound(2, h*100/total) +// +// def i = findFiledCount(cityid, 'latitude', sqlInstance) +// d['AJ'] = DataHandleUtils.getDoubleRound(2, i*100/total) +// +// def j = findFiledCount(cityid, 'parkingcnt', sqlInstance) +// d['AK'] = DataHandleUtils.getDoubleRound(2, j*100/total) +// println d // } -//},2) +//},1) + +//def fileName = "D:\\数据\\建行\\房产估值\\数据文件\\成交统计.xlsx" +def dealMap = findDealCount(sqlInstance) +ExcelUtils.multiThreadingModify(fileName,{d-> + if(d['C']){ + d['K'] = dealMap.get(d['C']) + } +},2) //def fileName = "d:\\数据\\建行\\房产估值\\经纬度错误数据修正.xlsx" @@ -170,7 +170,7 @@ ExcelUtils.multiThreadingModify(fileName,{d-> def findDealCount(sqlInstance){ - def sql = "select cityname,count(*) as count from bigdata_deal where dealdate>=to_date('2021-06-01','yyyy-mm-dd') and dealdate @@ -194,7 +194,7 @@ def findEstateType(cityid, sqlInstance){ } def findCount(cityid, sqlInstance){ - def sql = "select count(*) as count from bigdata_community where cityid='${cityid}'"; + def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}'"; def count = 0 sqlInstance.eachRow(sql,{r-> @@ -204,7 +204,7 @@ def findCount(cityid, sqlInstance){ } def findFiledCount(cityid, field, sqlInstance){ - def sql = "select count(*) as count from bigdata_community where cityid='${cityid}' and "+field+" is not null"; + def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}' and "+field+" is not null"; def count = 0 sqlInstance.eachRow(sql,{r-> count = r.count diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index f14ace3a9..8fadf19e0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -11,6 +11,8 @@ import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import java.util.concurrent.ConcurrentHashMap + /** * 建行模型估值项目 * 第三方均价数据同步 @@ -18,11 +20,13 @@ import com.mongodb.DBCollection @BatchOwner(Watcher.GaoJuan) class ThirdPartyPriceBatch extends BaseBatch { - def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def sDate= 202107//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def startDate = 202105 + def endDate = 202107 def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") - + DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") static main(args) { runBatch(new ThirdPartyPriceBatch(), args); @@ -33,13 +37,15 @@ class ThirdPartyPriceBatch extends BaseBatch { def citys = CcbCommonUtils.citys; def rows = [] def index = 0 + def weekend = DateUtils.getFirstDayOfWeek(new Date()) for(def city : citys){ index++ + def statMap = getDistrictStatByCity(city, weekend) def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"]) - def thirdPriceMap = findThirdPartyPrice(city) + def thirdPriceMap = findThirdPartyPrice(city, statMap) def rets = createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap) rows.addAll(rets) println index+" "+city+" "+rets.size() @@ -47,13 +53,13 @@ class ThirdPartyPriceBatch extends BaseBatch { - def fileName = "/home/goojia/ccb/third_party_price.txt" + def fileName = "/home/goojia/ccb/third_party_price.dat" if(CommonDatautils.isWindow()){ - fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\third_party_price.txt" + fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\third_party_price.dat" } - def header = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"] - FileUtils.batchCreateFile(rows, fileName, header, "utf-8", false); + def header = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"] + FileUtils.batchCreateFile(rows, fileName, header, "utf-8", true); } @@ -166,10 +172,10 @@ class ThirdPartyPriceBatch extends BaseBatch { } - def findThirdPartyPrice(def city){ + def findThirdPartyPrice(def city, def statMap){ def q = new BasicDBObject() q.put("city", city) - q.put("s_date", new BasicDBObject('$gte', sDate)) + q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate)) def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1)) def map = [:] for(def db : dbcursor){ @@ -242,13 +248,46 @@ class ThirdPartyPriceBatch extends BaseBatch { def resultMap = [:] map.each {key, value-> if(value && value.size()>1){ - resultMap.put(key, value) + if(value.size()==2 && !value.lianjia_esf_price && !value.beike_esf_price){ + def price = statMap.get(key) + if(price){ + def priceMap = [:] + def startPrice = (int)(price - price*0.3) + def endPrice = (int)(price*0.3 + price) + value.each{pk,pv-> + if(pk.contains("price")){ + if(pv>=startPrice && pv<=endPrice){ + priceMap.put(pk,pv) + } + }else{ + priceMap.put(pk,pv) + } + } + if(priceMap.size()>1){ + resultMap.put(key, priceMap) + } + } + }else{ + resultMap.put(key, value) + } + } } return resultMap } + def getDistrictStatByCity(city, weekend){ + def q = new BasicDBObject() + q.put("city", city) + q.put("weekend", weekend) + def dbcursor = DIST_SELL_STAT.find(q) + def map = new ConcurrentHashMap() + for(def d : dbcursor){ + map.put(d.region+"##"+d.district_name, d.avg_price) + } + return map + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index c2c545471..8a1af7d25 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -52,7 +52,7 @@ class YouDaDataParser { static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo -// task.esfParser();//友达二手房存入mongo + task.esfParser();//友达二手房存入mongo // task.newhouseToSaveEs();//新房保存到es task.esfhouseToSaveEs();//二手房保存到es // task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间 @@ -299,14 +299,14 @@ class YouDaDataParser { for(def i=0;i + def thirdPriceMap = findNearThirdPartyPrice(city) + def retMaps = findCreditHightDistrict3(city, "2021-08-22", thirdPriceMap) + def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","address","alias","estate_company", "developer","cubage_percent","household_count", "finish_building_date"]) + //基本情况合格的小区 + def distMap = [:] + for(def d : districts){ + // && d.household_count && d.household_count>=50 + def distKeys = retMaps.get(d.region) + if(distKeys && distKeys.contains(d.city+"##"+d.region+"##"+d.name)){ + if(d.finish_building_date){ + def year = CcbCodeUtils.parseBuildYear(d.finish_building_date) + if(year){ + def mod = 2021-year + if(mod <=15){ + d.house_year = mod + }else{ + d.house_year = null + } + }else{ + d.house_year = null + } + + }else{ + d.house_year = null + } + distMap.put(d.city+"##"+d.region+"##"+d.name, d) + } +// if(d.finish_building_date){ +// def year = CcbCodeUtils.parseBuildYear(d.finish_building_date) +// if(year){ +// def mod = 2021-year +// if(mod <=15){ +// def distKeys = retMaps.get(d.region) +// if(distKeys && distKeys.contains(d.city+"##"+d.region+"##"+d.name)){ +// d.house_year = mod +// distMap.put(d.city+"##"+d.region+"##"+d.name, d) +// } +// } +// } +// } + + } + println city +" 基本情况合格的小区:"+distMap.size() + + for(def r : regions){ + def names = retMaps.get(r) + if(names){ + def results = getRoomTypeAndTradeInfo(names, distMap, priceMap) + rets.addAll(results) + } + } + println "结束" + } + + println "开始写入Excel....."+rets.size() + ExcelUtils.write("d:\\数据\\广发\\楼盘(西安东莞杭州).xlsx", rets, "数据") + } + def getRoomTypeAndTradeInfo(def names, def distMap, def priceMap){ def rets = Collections.synchronizedList(new ArrayList()); @@ -332,4 +402,52 @@ class GuangfaBlankSelectDistrict { } return thirdPriceMap } + + + def findNearThirdPartyPrice(def city){ + def q = new BasicDBObject() + q.put("city", city) + q.put("s_date", new BasicDBObject('$gte', 202105)) + q.put("lianjia_esf_price", new BasicDBObject(QueryOperators.EXISTS, true)) + def dbcursor = AVERAGE_STATISTICS.find(q) + def thirdPriceMap = [:] + for(def d : dbcursor){ + def key = d.city+"#"+d.region+"#"+d.name + if(!thirdPriceMap.containsKey(key)){ + thirdPriceMap.put(d.city+"#"+d.region+"#"+d.name, d.lianjia_esf_price) + } + } + return thirdPriceMap + } + + def findCreditHightDistrict3(def city, def regions, def weekend, def thirdPriceMap){ + def q = new BasicDBObject() + q.put("city", city) + q.put("value_type", 2) + q.put("weekend", DateUtils.parseFromTightByPattern(weekend, "yyyy-MM-dd")) + def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("credit",-1)) + def retMaps = [:] + for(def d : dbcursor){ + def thirdPrice = thirdPriceMap.get(d.city +"#"+d.region+"#"+d.district_name) + if(thirdPrice){ + def precent = (thirdPrice-d.avg_price)*100/d.avg_price + if(Math.abs(precent)>=0 && Math.abs(precent)<=10){ + def rets = retMaps.get(d.region) + if(!rets){ + rets = [] + retMaps.put(d.region, rets) + } + rets << d.city+"##"+d.region+"##"+d.district_name + } + }else{ + def rets = retMaps.get(d.region) + if(!rets){ + rets = [] + retMaps.put(d.region, rets) + } + rets << d.city+"##"+d.region+"##"+d.district_name + } + } + return retMaps; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index 2b8980257..d0ebd044c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -15,9 +15,9 @@ class SynchDistrictPrice { def static final SHORT_DATE = Utils.getLastMonth() def thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); def districtDetailColl = CrawalDBUtil.getDistrictDetailColl(); -// def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房", -// "anjuke.com":"安居客","cityhouse.cn":"城市房产"] - def sourceMap = ["lianjia.com":"链家"] + def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房", + "anjuke.com":"安居客","cityhouse.cn":"城市房产"] +// def sourceMap = ["cityhouse.cn":"城市房产"] static main(args) { def task = new SynchDistrictPrice() task.run() @@ -28,23 +28,22 @@ class SynchDistrictPrice { def citys = CommonUtils.findAllCity() sourceMap.each { k, v -> - for(def city : citys){ - if(!["上海","北京","海口"].contains(city)){ - def rets = findCrawlDistrictDetails(city, k, v) - println city +" "+rets.size() - synchData(city, rets, v) - } + def rets = findCrawlDistrictDetails(city, k, v) + println city +" "+rets.size() + synchData(city, rets, v) } } } def findCrawlDistrictDetails(city, root, source){ + + def curMonthFirstDay = DateUtils.getFirstDayOfMonth(new Date()) def q = new BasicDBObject() q.put("city", city) q.put("root", root) - q.put('m_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2021-07-01"))); + q.put('m_date', new BasicDBObject('$gte', curMonthFirstDay)); def keys = new BasicDBObject() keys.put("city", 1) keys.put("region", 1) @@ -53,6 +52,7 @@ class SynchDistrictPrice { keys.put("address", 1) keys.put("avg_price", 1) keys.put("root", 1) + keys.put("c_date", 1) keys.put("url", 1) def rets = [] def dbcursor = districtDetailColl.find(q, keys) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy index f6887e506..10283440a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy @@ -14,8 +14,8 @@ class TestCrawlTotalCount { def run(){ def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房"] for(def source :sources){ - def count1 = findTotalCount(source, 202106); - def count2 = findTotalCount(source, 202107); + def count1 = findTotalCount(source, 202107); + def count2 = findTotalCount(source, 202108); println source +" "+count1 +" "+count2 } }