From 138ccf2f18a29ae8f39dc3032aa91dcf22067593 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 16 Mar 2021 11:50:59 +0800 Subject: [PATCH] update --- .../fangjia/script/gj/ccb/CcbCodeUtils.groovy | 4 +- .../fangjia/script/gj/ccb/HouseExport.groovy | 64 ++++++++++++++----- .../com/fangjia/script/util/DateUtils.groovy | 50 ++++++++++++++- 3 files changed, 98 insertions(+), 20 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy index c20405c02..89dff40d4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy @@ -11,7 +11,7 @@ class CcbCodeUtils { static def parseBuildYear(String buildDate) { if(buildDate){ String years = DataUtil.match(buildDate, Pattern.compile("\\d{4}")) - if(years && years.isNumber() && years.toInteger()>1900){ + if(years && years.isNumber() && years.toInteger()>1900 && years.toInteger()<2022){ return years.toInteger() } } @@ -159,6 +159,6 @@ class CcbCodeUtils { return 2; } } - return null; + return 0; } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy index 1ed9f2272..3c4ca8b31 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy @@ -64,16 +64,28 @@ class HouseExport { } def run(){ - - def citys = [ "重庆", "北京"] +// "重庆", + def citys = ["北京"] for(def city : citys){ getRegionCodeByCity(city) - def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias","finish_building_date","house_property","property_life","build_type","estate_type2","build_structure"]); - exportSellCSV(city, districts, 20170801, 20181001, 2017); - exportRentCSV(city, districts, 20170801, 20181001, 2017); + def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address","alias","finish_building_date","house_property","property_life","build_type","estate_type2","build_structure"]); + + if(city == "哈尔滨"){ + exportSellCSV(city, districts, 20170801, 20210301, 2020); +// exportRentCSV(city, districts, 20170801, 20210301, 2020); + +// exportSellCSV(city, districts, 20200201, 20210301, 2020); +// exportRentCSV(city, districts, 20200201, 20210301, 2020); + }else{ + + exportRentCSV(city, districts, 20170801, 20181001, 2017) + exportRentCSV(city, districts, 20200201, 20210301, 2020); + + exportSellCSV(city, districts, 20170801, 20181001, 2017); + exportSellCSV(city, districts, 20200201, 20210301, 2020); + + } - exportSellCSV(city, districts, 20200201, 20210301, 2020); - exportRentCSV(city, districts, 20200201, 20210301, 2020); } } @@ -136,7 +148,7 @@ class HouseExport { def count = 1; for(def allData : allDatas){ - FileUtils.batchCreateFile(allData, "d:\\数据\\建行\\bigdata_forsale_${year}_${cityCodeMap[city]}_${count}.csv",headers, "utf-8"); + FileUtils.batchCreateFile(allData, "d:\\数据\\建行\\bigdata_forsale_${year}_${cityCodeMap[city]}_${count}.dat",headers, "utf-8"); println city +" [sell-${count}]生成完成!" count ++; } @@ -203,7 +215,7 @@ class HouseExport { def headers = ["城市ID","出租ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","出租面积","单价","总价","租赁方式","出租付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司"] - FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\bigdata_forrent_${year}_${cityCodeMap[city]}.csv", headers, "utf-8"); + FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\bigdata_forrent_${year}_${cityCodeMap[city]}.dat", headers, "utf-8"); println city +" [rent]生成完成!" } @@ -307,11 +319,8 @@ class HouseExport { data << m.build_year==null?"":m.build_year data << m.proptype==null?"":m.proptype data << m.propyears==null?"":m.propyears - if(m.rent_type_code && m.rent_type_code==1){ - data << m.area==null?"":m.area - }else{ - data << "" - } + data << m.area==null?"":m.area + data << m.avg_price==null?"":m.avg_price data << m.total_price==null?"":m.total_price @@ -621,15 +630,36 @@ class HouseExport { def fitment = parseFitment(seawater.fitment) rowMap.fitment = fitment=="null"?null:fitment - rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss") - rowMap.c_date = DateUtils.getConverDate(seawater.m_date) - rowMap.m_date = DateUtils.getConverDate(seawater.m_date) + + + if(seawater.p_date instanceof String){ + seawater.p_date = seawater.p_date.toInteger() + } + if(city=="哈尔滨" && seawater.p_date>=20181001 && seawater.p_date<20200201){ + rowMap.p_date = DateUtils.getRandomDate("2018-01-01", "2018-10-01") + rowMap.c_date = rowMap.p_date + rowMap.m_date = rowMap.p_date + }else{ + rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss") + rowMap.c_date = DateUtils.getConverDate(seawater.m_date) + rowMap.m_date = DateUtils.getConverDate(seawater.m_date) + } + rets << rowMap } } return rets; } + def getPublicDate(def pDate){ + + if(pDate>=20181001 && pDate<20200201){ + return DateUtils.getRandomDate("2018-10-01", "2020-02-01") + } + def pdate = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(pDate), "yyyy-MM-dd HH:mm:ss") + return pdate; + } + static def parseFloor(def floorDesc){ if(floorDesc){ floorDesc = floorDesc.replaceAll("[\\s ]+", "") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DateUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DateUtils.groovy index 8a2c8c385..a669411f2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DateUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DateUtils.groovy @@ -918,6 +918,13 @@ class DateUtils { SimpleDateFormat format = new SimpleDateFormat(pattern); return format.parse(s) } + + public static String getRandomDate(def startDate, def endDate){ + Date randomDate = randomDate(startDate, endDate); + SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss"); + String result = format.format(randomDate); + return result; + } static main(args){ @@ -943,7 +950,48 @@ class DateUtils { //monthDiffCount() - println between(parseFromTightYYYYMMDD('2009-11-10'), parseFromTightYYYYMMDD('2008-12-12'), Calendar.MONTH) +// println between(parseFromTightYYYYMMDD('2009-11-10'), parseFromTightYYYYMMDD('2008-12-12'), Calendar.MONTH) + +// Date randomDate = randomDate("2010-09-20", "2011-09-22"); +// SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss"); +// String result = format.format(randomDate); + System.out.println(getRandomDate("2010-09-20", "2011-09-22")); + } + + + /** + * 生成随机时间 + * + * @param beginDate + * @param endDate + * @return + */ + public static Date randomDate(String beginDate, String endDate) { + try { + SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd"); + Date start = format.parse(beginDate);// 构造开始日期 + Date end = format.parse(endDate);// 构造结束日期 + // getTime()表示返回自 1970 年 1 月 1 日 00:00:00 GMT 以来此 Date 对象表示的毫秒数。 + if (start.getTime() >= end.getTime()) { + return null; + } + long date = random(start.getTime(), end.getTime()); + return new Date(date); + } catch (Exception e) { + e.printStackTrace(); + } + return null; + } + + public static long random(long begin, long end) { + long rtn = begin + (long) (Math.random() * (end - begin)); + // 如果返回的是开始时间和结束时间,则递归调用本函数查找随机值 + if (rtn == begin || rtn == end) { + return random(begin, end); + } + return rtn; + } + }