This commit is contained in:
2022-06-20 19:28:59 +08:00
parent 6364ce61b1
commit 10726b5d5e
5 changed files with 46 additions and 50 deletions
@@ -56,47 +56,43 @@ class BigdataDealBatch extends BaseBatch{
def ccbCitys = CcbCommonUtils.citys;
for(def city : citys){
if(!ccbCitys.contains(city)){
// if(!ccbCitys.contains(city)){
addDealPriceMatch(city);
}
}
//// println citys.unique().size()
// def rows = []
// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
// for(def city : citys){
// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
// def ppeDists = [:]
//// if(["贵阳","佛山","杭州","沈阳","天津","中山","徐州","绍兴","南通","哈尔滨","惠州","长沙","青岛","长春","烟台","温州","太原","台州","石家庄","宁波","常州","大连","嘉兴","金华","昆明"].contains(city)){
//// ppeDists = getPpeGTE15Datas(city)
//// }
//
// def statMap = [:]//getDistrictStatByCity(city, weekend)
// if(city=="上海"){
// def youDataTrade = getYouDaData(statMap)
// def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
// rows.addAll(youdaDatas)
// println city +" "+youdaDatas.size()
// }
//
//
// def trades = getCityData(city, statMap, ppeDists)
// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
// rows.addAll(rets)
// println city +" 成交共:"+rets.size()
//
// def cityDatas = []
// cityDatas << headers
// cityDatas.addAll(rows)
// }
//
// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
// if(CommonDatautils.isWindow()){
// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
// }
//
// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
// println "txt 生成"
}
def rows = []
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
for(def city : citys){
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def ppeDists = [:]
def statMap = [:]//getDistrictStatByCity(city, weekend)
if(city=="上海"){
def youDataTrade = getYouDaData(statMap)
def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
rows.addAll(youdaDatas)
println city +" "+youdaDatas.size()
}
def trades = getCityData(city, statMap, ppeDists)
def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
rows.addAll(rets)
println city +" 成交共:"+rets.size()
def cityDatas = []
cityDatas << headers
cityDatas.addAll(rows)
}
def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
}
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
println "txt 生成"
}
@@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap
@BatchOwner(Watcher.GaoJuan)
class ThirdPartyPriceBatch extends BaseBatch {
def sDate= 202204//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def startDate = 202202
def endDate = 202204
def sDate= 202205//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def startDate = 202203
def endDate = 202205
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
@@ -75,13 +75,13 @@ class TradeStatLog {
def run(){
def citys = CcbCommonUtils.citys;
def rets = []
rets << ["城市","2月总量","3月总量","4月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
rets << ["城市","3月总量","4月总量","5月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"]
//def startDate = "2021-05-01"
def startDate = "2022-02-01"
def date = "2022-03-01"
def endDate = "2022-04-01"
def newDate = "2022-05-01"
def startDate = "2022-03-01"
def date = "2022-04-01"
def endDate = "2022-05-01"
def newDate = "2022-06-01"
for(def city : citys){
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
@@ -131,7 +131,7 @@ class TradeStatLog {
}
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-04成交统计.xlsx", rets, "数据")
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-05成交统计.xlsx", rets, "数据")
println "表格生成完成"
}
@@ -34,7 +34,7 @@ public class LianjiaCrawlByChrome {
static main(args) {
def task = new LianjiaCrawlByChrome();
task.run()
// task.run()
// task.run1()
task.addDealRegion();
task.standardRegion()
@@ -40,7 +40,7 @@ class DataStat {
def run(){
def source = "58同城"//
def source = "贝壳找房"//
def citys = findCrawlCity(source, 202202);
for(def city : citys){
def count1 = findCount(source, city, 202204);