From 10726b5d5e49907cbb27f2c9c20bea506ecc9234 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 20 Jun 2022 19:28:59 +0800 Subject: [PATCH] update --- .../ccbevaluation/BigdataDealBatch.groovy | 74 +++++++++---------- .../ccbevaluation/ThirdPartyPriceBatch.groovy | 6 +- .../script/ccbevaluation/TradeStatLog.groovy | 12 +-- .../gj/paimai/LianjiaCrawlByChrome.groovy | 2 +- .../script/thirdPriceCrawl/DataStat.groovy | 2 +- 5 files changed, 46 insertions(+), 50 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index f606cd13f..c1ea41780 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -56,47 +56,43 @@ class BigdataDealBatch extends BaseBatch{ def ccbCitys = CcbCommonUtils.citys; for(def city : citys){ - if(!ccbCitys.contains(city)){ +// if(!ccbCitys.contains(city)){ addDealPriceMatch(city); - } - } -//// println citys.unique().size() -// def rows = [] -// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] -// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() -// for(def city : citys){ -// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) -// def ppeDists = [:] -//// if(["贵阳","佛山","杭州","沈阳","天津","中山","徐州","绍兴","南通","哈尔滨","惠州","长沙","青岛","长春","烟台","温州","太原","台州","石家庄","宁波","常州","大连","嘉兴","金华","昆明"].contains(city)){ -//// ppeDists = getPpeGTE15Datas(city) -//// } -// -// def statMap = [:]//getDistrictStatByCity(city, weekend) -// if(city=="上海"){ -// def youDataTrade = getYouDaData(statMap) -// def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda") -// rows.addAll(youdaDatas) -// println city +" "+youdaDatas.size() // } -// -// -// def trades = getCityData(city, statMap, ppeDists) -// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") -// rows.addAll(rets) -// println city +" 成交共:"+rets.size() -// -// def cityDatas = [] -// cityDatas << headers -// cityDatas.addAll(rows) -// } -// -// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" -// if(CommonDatautils.isWindow()){ -// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" -// } -// -// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); -// println "txt 生成" + } + def rows = [] + def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + for(def city : citys){ + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + def ppeDists = [:] + + def statMap = [:]//getDistrictStatByCity(city, weekend) + if(city=="上海"){ + def youDataTrade = getYouDaData(statMap) + def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda") + rows.addAll(youdaDatas) + println city +" "+youdaDatas.size() + } + + + def trades = getCityData(city, statMap, ppeDists) + def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") + rows.addAll(rets) + println city +" 成交共:"+rets.size() + + def cityDatas = [] + cityDatas << headers + cityDatas.addAll(rows) + } + + def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" + if(CommonDatautils.isWindow()){ + fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" + } + + FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); + println "txt 生成" } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index d1cc4209b..fcfd4c0bd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap @BatchOwner(Watcher.GaoJuan) class ThirdPartyPriceBatch extends BaseBatch { - def sDate= 202204//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); - def startDate = 202202 - def endDate = 202204 + def sDate= 202205//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def startDate = 202203 + def endDate = 202205 def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index 3b3e6a7c5..fc73e4867 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -75,13 +75,13 @@ class TradeStatLog { def run(){ def citys = CcbCommonUtils.citys; def rets = [] - rets << ["城市","2月总量","3月总量","4月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] + rets << ["城市","3月总量","4月总量","5月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"] //def startDate = "2021-05-01" - def startDate = "2022-02-01" - def date = "2022-03-01" - def endDate = "2022-04-01" - def newDate = "2022-05-01" + def startDate = "2022-03-01" + def date = "2022-04-01" + def endDate = "2022-05-01" + def newDate = "2022-06-01" for(def city : citys){ def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) @@ -131,7 +131,7 @@ class TradeStatLog { } - ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-04成交统计.xlsx", rets, "数据") + ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-05成交统计.xlsx", rets, "数据") println "表格生成完成" } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index f94fd7fc7..ebe6f911a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -34,7 +34,7 @@ public class LianjiaCrawlByChrome { static main(args) { def task = new LianjiaCrawlByChrome(); - task.run() +// task.run() // task.run1() task.addDealRegion(); task.standardRegion() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy index aa363f85b..ec9d7f64b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy @@ -40,7 +40,7 @@ class DataStat { def run(){ - def source = "58同城"// + def source = "贝壳找房"// def citys = findCrawlCity(source, 202202); for(def city : citys){ def count1 = findCount(source, city, 202204);