From ca4edb2bbc3041b23a0ca91bd7ae3cd1603e1710 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sun, 14 Mar 2021 11:52:58 +0800 Subject: [PATCH 1/5] update --- .../fangjia/script/gj/ccb/FileUtils.groovy | 43 +++++++++++++++++++ .../fangjia/script/gj/ccb/HouseExport.groovy | 38 +--------------- 2 files changed, 44 insertions(+), 37 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy new file mode 100644 index 000000000..8c895b5c0 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy @@ -0,0 +1,43 @@ +package com.fangjia.script.gj.ccb + +import org.apache.commons.lang.StringUtils + +class FileUtils { + static def batchCreateFile(ret, fileName, headers,charset){ + File file = new File(fileName); + if (file.exists()) { + file.delete(); + } + + OutputStream out = new FileOutputStream(fileName); + def sb = new StringBuffer(); + + sb.append(headers.join("@|@")) + sb.append(System.getProperty("line.separator")); + + out.write(sb.toString().getBytes(charset)); + sb.setLength(0) + + def index = 0 + for(def row : ret){ + index++ + sb.append(StringUtils.join(row.toArray(), "|@|")) + sb.append(System.getProperty("line.separator")); + println index + + if(index%10000==0){ + println index+"开始写入CSV "+new Date().toLocaleString() + out.write(sb.toString().getBytes(charset)); + sb.setLength(0) + println index+"结束写入CSV "+new Date().toLocaleString() + } + + } + + if(sb.length()>0){ + out.write(sb.toString().getBytes(charset)); + sb.setLength(0) + } + out.close(); + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy index a3bad5266..27f1870bb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy @@ -65,7 +65,7 @@ class HouseExport { def run(){ - def city = "重庆" + def city = "北京" getRegionCodeByCity(city) def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias","finish_building_date","house_property","property_life","build_type","estate_type2","build_structure"]); @@ -318,43 +318,7 @@ class HouseExport { } - def batchCreateFile(ret, fileName, headers,charset){ - File file = new File(fileName); - if (file.exists()) { - file.delete(); - } - OutputStream out = new FileOutputStream(fileName); - def sb = new StringBuffer(); - - sb.append(headers.join("@|@")) - sb.append(System.getProperty("line.separator")); - - out.write(sb.toString().getBytes(charset)); - sb.setLength(0) - - def index = 0 - for(def row : ret){ - index++ - sb.append(StringUtils.join(row.toArray(), "|@|")) - sb.append(System.getProperty("line.separator")); - println index - - if(index%10000==0){ - println index+"开始写入CSV "+new Date().toLocaleString() - out.write(sb.toString().getBytes(charset)); - sb.setLength(0) - println index+"结束写入CSV "+new Date().toLocaleString() - } - - } - - if(sb.length()>0){ - out.write(sb.toString().getBytes(charset)); - sb.setLength(0) - } - out.close(); - } From 1d4318142b54bea18234a2bc7b684bf96ae70d30 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sun, 14 Mar 2021 16:05:52 +0800 Subject: [PATCH 2/5] update --- .../fangjia/script/gj/ccb/FileUtils.groovy | 2 +- .../fangjia/script/gj/ccb/HouseExport.groovy | 44 ++++++++++++++----- 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy index 8c895b5c0..09f80b9da 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy @@ -12,7 +12,7 @@ class FileUtils { OutputStream out = new FileOutputStream(fileName); def sb = new StringBuffer(); - sb.append(headers.join("@|@")) + sb.append(headers.join("|@|")) sb.append(System.getProperty("line.separator")); out.write(sb.toString().getBytes(charset)); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy index 27f1870bb..7027cf373 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy @@ -76,6 +76,8 @@ class HouseExport { } def exportSellCSV(city, districts){ + + final CountDownLatch latch = new CountDownLatch(districts.size()) println city +" 小区量:"+districts.size() def datas = [] @@ -124,11 +126,16 @@ class HouseExport { }) } latch.await() - println " 去重后:"+retMap.values() - def allDatas = createSellDatas(retMap.values()); + def allDatas = createSellDatas(retMap); def headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司"] - batchCreateFile(allDatas, "d:\\数据\\建行\\bigdata_forsale_${cityCodeMap[city]}.csv",headers, "utf-8"); - println city +" [sell]生成完成!" + + def count = 1; + for(def allData : allDatas){ + FileUtils.batchCreateFile(allData, "d:\\数据\\建行\\bigdata_forsale_2017_${cityCodeMap[city]}_${count}.csv",headers, "utf-8"); + println city +" [sell-${count}]生成完成!" + count ++; + } + } @@ -184,17 +191,19 @@ class HouseExport { }) } latch.await() - println " 去重后:"+retMap.values() - def allDatas = createRentDatas(retMap.values()); + + def allDatas = createRentDatas(retMap); def headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","出租面积","单价","总价","租赁方式","出租付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司"] - batchCreateFile(allDatas, "d:\\数据\\建行\\bigdata_forrent_${cityCodeMap[city]}.csv", headers, "utf-8"); + FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\bigdata_forrent_${cityCodeMap[city]}.csv", headers, "utf-8"); println city +" [rent]生成完成!" } - def createSellDatas(def list){ + def createSellDatas(def map){ + def datas1 = [] + def datas2 = [] def datas = [] - for(def m : list){ + map.each{k,m-> def data = [] data << m.city_code data << m.houseId @@ -248,15 +257,26 @@ class HouseExport { data << m.m_date==null?"":m.m_date data << m.source - datas << data + if(datas1.size()>2000000){ + datas2 << data + }else{ + datas1 << data + } + } + if(datas1.size()>0){ + datas << datas1 + } + + if(datas2.size()>0){ + datas << datas2 } return datas ; } - def createRentDatas(def list){ + def createRentDatas(def map){ def datas = [] - for(def m : list){ + map.each{k,m-> def data = [] data << m.city_code data << m.houseId From 7be22202e3ce9b5cbac3ef153e7260f80bfa8dae Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 15 Mar 2021 00:16:59 +0800 Subject: [PATCH 3/5] update --- .../fangjia/script/gj/ccb/CcbCodeUtils.groovy | 2 +- .../fangjia/script/gj/ccb/HouseExport.groovy | 82 +++++++++++-------- .../com/fangjia/script/util/DateUtils.groovy | 16 ++-- 3 files changed, 59 insertions(+), 41 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy index e3cb5bfee..c20405c02 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy @@ -11,7 +11,7 @@ class CcbCodeUtils { static def parseBuildYear(String buildDate) { if(buildDate){ String years = DataUtil.match(buildDate, Pattern.compile("\\d{4}")) - if(years){ + if(years && years.isNumber() && years.toInteger()>1900){ return years.toInteger() } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy index 7027cf373..3dbeebc2f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy @@ -11,6 +11,7 @@ import com.fangjia.script.util.ExcelUtils import com.mongodb.BasicDBObject import com.mongodb.DBObject import org.apache.commons.lang.StringUtils +import org.bson.types.ObjectId import org.elasticsearch.index.query.BoolQueryBuilder import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilders @@ -51,7 +52,7 @@ class HouseExport { public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) - static def totalCount = new AtomicLong() + def cityCodeMap = ["北京":"110100","重庆":"500100", "哈尔滨":"230100"] def regionCodeMap = [:] @@ -60,24 +61,25 @@ class HouseExport { def task = new HouseExport(); task.run() - } def run(){ - def city = "北京" - getRegionCodeByCity(city) - def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias","finish_building_date","house_property","property_life","build_type","estate_type2","build_structure"]); - - - exportSellCSV(city, districts); -// exportRentCSV(city, districts); + def citys = ["北京", "哈尔滨", "重庆"] + for(def city : citys){ + getRegionCodeByCity(city) + def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias","finish_building_date","house_property","property_life","build_type","estate_type2","build_structure"]); + exportSellCSV(city, districts, 20170801, 20180901, 2017); + exportRentCSV(city, districts, 20170801, 20180901, 2017); + exportSellCSV(city, districts, 20200201, 20210201, 2020); + exportRentCSV(city, districts, 20200201, 20210201, 2020); + } } - def exportSellCSV(city, districts){ - + def exportSellCSV(city, districts, startDate, endDate, year){ + def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(districts.size()) println city +" 小区量:"+districts.size() def datas = [] @@ -89,7 +91,7 @@ class HouseExport { void run() { try { totalCount.addAndGet(1) - def ret = findHouses(city, "sell", 20170801,20180901, rMap.name, rMap.alias) + def ret = findHouses(city, "sell", startDate,endDate, rMap.name, rMap.alias) if(ret){ def cityCode = cityCodeMap.get(city) def regionCode = regionCodeMap.get(city +"-"+rMap.region) @@ -131,7 +133,7 @@ class HouseExport { def count = 1; for(def allData : allDatas){ - FileUtils.batchCreateFile(allData, "d:\\数据\\建行\\bigdata_forsale_2017_${cityCodeMap[city]}_${count}.csv",headers, "utf-8"); + FileUtils.batchCreateFile(allData, "d:\\数据\\建行\\bigdata_forsale_${year}_${cityCodeMap[city]}_${count}.csv",headers, "utf-8"); println city +" [sell-${count}]生成完成!" count ++; } @@ -140,10 +142,10 @@ class HouseExport { } - def exportRentCSV(city, districts){ + def exportRentCSV(city, districts, startDate, endDate, year){ final CountDownLatch latch = new CountDownLatch(districts.size()) println city +" 小区量:"+districts.size() - + def totalCount = new AtomicLong() def retMap = new ConcurrentHashMap() for(def i=0;i def data = [] data << m.city_code - data << m.houseId + //data << m.houseId + data << new ObjectId() data << "" data << m.city data << m.region_code @@ -279,7 +283,8 @@ class HouseExport { map.each{k,m-> def data = [] data << m.city_code - data << m.houseId +// data << m.houseId + data << new ObjectId() data << "" data << m.city data << m.region_code @@ -479,7 +484,7 @@ class HouseExport { QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false); QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); - QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("cat", "sell")) + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("cat", cat)) .must(date_range).must(area_range) if(!alias){ @@ -541,11 +546,17 @@ class HouseExport { floor = null seawater.height = null } + if(floor && floor.toInteger()<1){ + floor = null + } } def rowMap = [:]; if(cat == "rent"){ rowMap.rent_type = seawater.rent_type + if(seawater.avg_price && seawater.avg_price>1000){ + continue + } } rowMap.houseId = seawater.id rowMap.city = city @@ -557,6 +568,9 @@ class HouseExport { rowMap.total_price = seawater.total_price rowMap.avg_price = seawater.avg_price rowMap.height = seawater.height=="null"?null:seawater.height + if(rowMap.height && rowMap.height<1){ + rowMap.height = null + } rowMap.floor = floor=="null"?null:floor rowMap.room = seawater.room=="null"?null:seawater.room rowMap.hall = seawater.hall=="null"?null:seawater.hall @@ -591,18 +605,18 @@ class HouseExport { if (matcher1.find()) { return matcher1.group(1) } - if(floorDesc.contains("中楼层") || floorDesc.contains("中层")){ - return "中层" - } - - if(floorDesc.contains("高楼层") || floorDesc.contains("高层")){ - return "高层" - } - - - if(floorDesc.contains("低楼层") || floorDesc.contains("低层")){ - return "低层" - } +// if(floorDesc.contains("中楼层") || floorDesc.contains("中层")){ +// return "中层" +// } +// +// if(floorDesc.contains("高楼层") || floorDesc.contains("高层")){ +// return "高层" +// } +// +// +// if(floorDesc.contains("低楼层") || floorDesc.contains("低层")){ +// return "低层" +// } if(floorDesc.isNumber()){ return floorDesc; } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DateUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DateUtils.groovy index 8bc5d9198..8a2c8c385 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DateUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DateUtils.groovy @@ -767,15 +767,19 @@ class DateUtils { } static String getConverDate(def mills){ - String s = null + Date date = null; if(mills instanceof String){ - s = mills.replace("T"," ").replace(".000Z","") + date = com.fangjia.core.util.DateUtil.parseISODate(mills) + Calendar cal = Calendar.getInstance(); + cal.setTime(date); + cal.add(Calendar.HOUR, 8);// 24小时制 + date = cal.getTime(); + //s = mills.replace("T"," ").replace(".000Z","") }else if(mills instanceof Long){ - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - Date date = new Date(mills); - s = dateFormat.format(date); + date = new Date(mills) } - + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + String s = dateFormat.format(date); return s } From 592c8642333a38379932b59e8fc470d49da8488e Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 15 Mar 2021 22:20:41 +0800 Subject: [PATCH 4/5] update --- .../fangjia/script/gj/ccb/HouseExport.groovy | 83 ++++++-- .../script/gj/test/AnjukeCrawlToDB.groovy | 199 ++++++++++++++++++ 2 files changed, 264 insertions(+), 18 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/AnjukeCrawlToDB.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy index 3dbeebc2f..1ed9f2272 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy @@ -65,15 +65,15 @@ class HouseExport { def run(){ - def citys = ["北京", "哈尔滨", "重庆"] + def citys = [ "重庆", "北京"] for(def city : citys){ getRegionCodeByCity(city) def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias","finish_building_date","house_property","property_life","build_type","estate_type2","build_structure"]); - exportSellCSV(city, districts, 20170801, 20180901, 2017); - exportRentCSV(city, districts, 20170801, 20180901, 2017); + exportSellCSV(city, districts, 20170801, 20181001, 2017); + exportRentCSV(city, districts, 20170801, 20181001, 2017); - exportSellCSV(city, districts, 20200201, 20210201, 2020); - exportRentCSV(city, districts, 20200201, 20210201, 2020); + exportSellCSV(city, districts, 20200201, 20210301, 2020); + exportRentCSV(city, districts, 20200201, 20210301, 2020); } } @@ -87,11 +87,12 @@ class HouseExport { def retMap = new ConcurrentHashMap() for(def i=0;i=5 || seawater.room<1)){ - continue; + if(!seawater.room){ + seawater.room = null + } + + if(seawater.room){ + if(seawater.room>=5 || seawater.room<1){ + continue; + } } if(!seawater.hall){ @@ -540,15 +568,21 @@ class HouseExport { continue; } + def floor = parseFloor(seawater.floor_info) if(seawater.height && floor && floor.isNumber()){ if(floor.toInteger()>seawater.height){ floor = null seawater.height = null } - if(floor && floor.toInteger()<1){ - floor = null - } + + } + + if(!floor){ + seawater.floor = null + } + if(floor && floor.toInteger()<1){ + floor = null } def rowMap = [:]; @@ -557,11 +591,16 @@ class HouseExport { if(seawater.avg_price && seawater.avg_price>1000){ continue } + + if(seawater.avg_price && seawater.avg_price<10){ + continue + } + } rowMap.houseId = seawater.id rowMap.city = city - rowMap.region =seawater.region - rowMap.address = seawater.address + //rowMap.region =seawater.region + //rowMap.address = seawater.address rowMap.area = seawater.area @@ -698,6 +737,14 @@ class HouseExport { } + /* + * 生成一个用不重复的ID + */ +def GenNonDuplicateID(randomLength){ + return Number(Math.random().toString().substr(3,randomLength) + Date.now()).toString(36) +} +//GenNonDuplicateID()将生成 rfmipbs8ag0kgkcogc 类似的ID + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/AnjukeCrawlToDB.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/AnjukeCrawlToDB.groovy new file mode 100644 index 000000000..8c5009841 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/AnjukeCrawlToDB.groovy @@ -0,0 +1,199 @@ +package com.fangjia.script.gj.test + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import org.bson.types.ObjectId + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class AnjukeCrawlToDB { + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + + def totalCount = new AtomicLong() + + static main(args) { + def task = new AnjukeCrawlToDB(); + +// task.run() + + task.stat() + } + + def stat(){ + def districts = CommonDatautils.findAllDistricts("北京", "district", ["estate_charge","finish_building_date","build_area","household_count","car_site_count","cubage_percent","virescence_percent","estate_company","developer"]); + + def total = districts.size() + def estate_chargeCount = 0 + def build_areaCount = 0 + def household_countCount = 0 + def finish_building_dateCount = 0 + def car_site_countCount = 0 + def cubage_percentCount = 0 + def virescence_percentCount = 0 + def developerCount = 0 + def estate_companyCount = 0 + for(def dists : districts){ + if(dists.estate_charge){ + estate_chargeCount++ + } + // 总建面积 + if(dists.build_area){ + build_areaCount++ + } + // 总数 + if(dists.household_count){ + household_countCount++ + } + // 建造年代 + if(dists.finish_building_date){ + finish_building_dateCount++ + } + // 停车位 + if(dists.car_site_count){ + car_site_countCount++ + } + // 容积率 + if(dists.cubage_percent){ + cubage_percentCount++ + } + // 绿化率 + if(dists.virescence_percent){ + virescence_percentCount++ + } + // 开发商 + if(dists.developer){ + developerCount++ + } + // 物业公司 + if(dists.estate_company){ + estate_companyCount++ + } + } + + + def estate_charge_pre = DataHandleUtils.getDoubleRound(2, estate_chargeCount*100/total) + def build_area_pre = DataHandleUtils.getDoubleRound(2, build_areaCount*100/total) + def household_count_pre = DataHandleUtils.getDoubleRound(2, household_countCount*100/total) + def finish_building_date_pre = DataHandleUtils.getDoubleRound(2, finish_building_dateCount*100/total) + def car_site_pre = DataHandleUtils.getDoubleRound(2, car_site_countCount*100/total) + def cubage_percent_pre = DataHandleUtils.getDoubleRound(2, cubage_percentCount*100/total) + def virescence_percent_pre = DataHandleUtils.getDoubleRound(2, virescence_percentCount*100/total) + def developer_pre = DataHandleUtils.getDoubleRound(2, developerCount*100/total) + def estate_company_pre = DataHandleUtils.getDoubleRound(2, estate_companyCount*100/total) + + println finish_building_date_pre + println virescence_percent_pre + println cubage_percent_pre + + println build_area_pre + println estate_charge_pre + println developer_pre + println estate_company_pre +// println household_count_pre +// println car_site_pre + + + + + } + + + + + def run(){ + def datas = ExcelUtils.read("D:\\数据\\待抓取小区_安居客_detail.xlsx",0,1) + + + final CountDownLatch latch = new CountDownLatch(datas.size()) + for(def i=0;i0){ + udata.put("estate_charge",estateCharge) + } + } + // 总建面积 + if(!dists.build_area && data['L']){ + def buildArea = data['L'].toString().replace("m?","") + if(buildArea){ + udata.put("build_area",buildArea.toDouble().toInteger().toString()) + } + + } + // 总数 + if(!dists.household_count && data['M']){ + def householdCount= data['M'] + if(householdCount && householdCount.toString().isNumber()){ + udata.put("household_count",householdCount.toInteger()) + } + } + // 建造年代 + if(!dists.finish_building_date && data['N']){ + def year = data['N'].replace("年","") + if(year && year.isNumber()){ + udata.put("finish_building_date",year.toString()) + } + } + // 停车位 + if(!dists.car_site_count && data['O']){ + if(data['O'].isNumber()){ + udata.put("car_site_count", data['O'].toInteger()) + } + } + // 容积率 + if(!dists.cubage_percent && data['R']){ + def cubage_percent = data['R'] + if(cubage_percent && cubage_percent.isNumber() && cubage_percent.toDouble()<10){ + udata.put("cubage_percent", cubage_percent.toString()) + } + } + // 绿化率 + if(!dists.virescence_percent && data['Q']){ + def virescence_percent = data['Q'].toString().replace("%","") + if(virescence_percent && virescence_percent.isNumber()){ + udata.put("virescence_percent", virescence_percent.toString()) + } + } + // 开发商 + if(!dists.developer && data['R'] && !data['R'].contains("未知") && !data['R'].contains("无")){ + udata.put("developer", data['R']) + } + // 物业公司 + if(!dists.estate_company && data['S'] && !data['S'].contains("未知") && !data['S'].contains("无")){ + udata.put("estate_company",data['S']) + } + + if(udata){ + udata.put("m_date", new Date()) + println totalCount.get()+" "+data['A']+" "+udata + DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(data['A'])),new BasicDBObject('$set', udata)); + } + + } + } +} From 138ccf2f18a29ae8f39dc3032aa91dcf22067593 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 16 Mar 2021 11:50:59 +0800 Subject: [PATCH 5/5] update --- .../fangjia/script/gj/ccb/CcbCodeUtils.groovy | 4 +- .../fangjia/script/gj/ccb/HouseExport.groovy | 64 ++++++++++++++----- .../com/fangjia/script/util/DateUtils.groovy | 50 ++++++++++++++- 3 files changed, 98 insertions(+), 20 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy index c20405c02..89dff40d4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy @@ -11,7 +11,7 @@ class CcbCodeUtils { static def parseBuildYear(String buildDate) { if(buildDate){ String years = DataUtil.match(buildDate, Pattern.compile("\\d{4}")) - if(years && years.isNumber() && years.toInteger()>1900){ + if(years && years.isNumber() && years.toInteger()>1900 && years.toInteger()<2022){ return years.toInteger() } } @@ -159,6 +159,6 @@ class CcbCodeUtils { return 2; } } - return null; + return 0; } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy index 1ed9f2272..3c4ca8b31 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy @@ -64,16 +64,28 @@ class HouseExport { } def run(){ - - def citys = [ "重庆", "北京"] +// "重庆", + def citys = ["北京"] for(def city : citys){ getRegionCodeByCity(city) - def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias","finish_building_date","house_property","property_life","build_type","estate_type2","build_structure"]); - exportSellCSV(city, districts, 20170801, 20181001, 2017); - exportRentCSV(city, districts, 20170801, 20181001, 2017); + def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address","alias","finish_building_date","house_property","property_life","build_type","estate_type2","build_structure"]); + + if(city == "哈尔滨"){ + exportSellCSV(city, districts, 20170801, 20210301, 2020); +// exportRentCSV(city, districts, 20170801, 20210301, 2020); + +// exportSellCSV(city, districts, 20200201, 20210301, 2020); +// exportRentCSV(city, districts, 20200201, 20210301, 2020); + }else{ + + exportRentCSV(city, districts, 20170801, 20181001, 2017) + exportRentCSV(city, districts, 20200201, 20210301, 2020); + + exportSellCSV(city, districts, 20170801, 20181001, 2017); + exportSellCSV(city, districts, 20200201, 20210301, 2020); + + } - exportSellCSV(city, districts, 20200201, 20210301, 2020); - exportRentCSV(city, districts, 20200201, 20210301, 2020); } } @@ -136,7 +148,7 @@ class HouseExport { def count = 1; for(def allData : allDatas){ - FileUtils.batchCreateFile(allData, "d:\\数据\\建行\\bigdata_forsale_${year}_${cityCodeMap[city]}_${count}.csv",headers, "utf-8"); + FileUtils.batchCreateFile(allData, "d:\\数据\\建行\\bigdata_forsale_${year}_${cityCodeMap[city]}_${count}.dat",headers, "utf-8"); println city +" [sell-${count}]生成完成!" count ++; } @@ -203,7 +215,7 @@ class HouseExport { def headers = ["城市ID","出租ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","出租面积","单价","总价","租赁方式","出租付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司"] - FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\bigdata_forrent_${year}_${cityCodeMap[city]}.csv", headers, "utf-8"); + FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\bigdata_forrent_${year}_${cityCodeMap[city]}.dat", headers, "utf-8"); println city +" [rent]生成完成!" } @@ -307,11 +319,8 @@ class HouseExport { data << m.build_year==null?"":m.build_year data << m.proptype==null?"":m.proptype data << m.propyears==null?"":m.propyears - if(m.rent_type_code && m.rent_type_code==1){ - data << m.area==null?"":m.area - }else{ - data << "" - } + data << m.area==null?"":m.area + data << m.avg_price==null?"":m.avg_price data << m.total_price==null?"":m.total_price @@ -621,15 +630,36 @@ class HouseExport { def fitment = parseFitment(seawater.fitment) rowMap.fitment = fitment=="null"?null:fitment - rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss") - rowMap.c_date = DateUtils.getConverDate(seawater.m_date) - rowMap.m_date = DateUtils.getConverDate(seawater.m_date) + + + if(seawater.p_date instanceof String){ + seawater.p_date = seawater.p_date.toInteger() + } + if(city=="哈尔滨" && seawater.p_date>=20181001 && seawater.p_date<20200201){ + rowMap.p_date = DateUtils.getRandomDate("2018-01-01", "2018-10-01") + rowMap.c_date = rowMap.p_date + rowMap.m_date = rowMap.p_date + }else{ + rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss") + rowMap.c_date = DateUtils.getConverDate(seawater.m_date) + rowMap.m_date = DateUtils.getConverDate(seawater.m_date) + } + rets << rowMap } } return rets; } + def getPublicDate(def pDate){ + + if(pDate>=20181001 && pDate<20200201){ + return DateUtils.getRandomDate("2018-10-01", "2020-02-01") + } + def pdate = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(pDate), "yyyy-MM-dd HH:mm:ss") + return pdate; + } + static def parseFloor(def floorDesc){ if(floorDesc){ floorDesc = floorDesc.replaceAll("[\\s ]+", "") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DateUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DateUtils.groovy index 8a2c8c385..a669411f2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DateUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DateUtils.groovy @@ -918,6 +918,13 @@ class DateUtils { SimpleDateFormat format = new SimpleDateFormat(pattern); return format.parse(s) } + + public static String getRandomDate(def startDate, def endDate){ + Date randomDate = randomDate(startDate, endDate); + SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss"); + String result = format.format(randomDate); + return result; + } static main(args){ @@ -943,7 +950,48 @@ class DateUtils { //monthDiffCount() - println between(parseFromTightYYYYMMDD('2009-11-10'), parseFromTightYYYYMMDD('2008-12-12'), Calendar.MONTH) +// println between(parseFromTightYYYYMMDD('2009-11-10'), parseFromTightYYYYMMDD('2008-12-12'), Calendar.MONTH) + +// Date randomDate = randomDate("2010-09-20", "2011-09-22"); +// SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss"); +// String result = format.format(randomDate); + System.out.println(getRandomDate("2010-09-20", "2011-09-22")); + } + + + /** + * 生成随机时间 + * + * @param beginDate + * @param endDate + * @return + */ + public static Date randomDate(String beginDate, String endDate) { + try { + SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd"); + Date start = format.parse(beginDate);// 构造开始日期 + Date end = format.parse(endDate);// 构造结束日期 + // getTime()表示返回自 1970 年 1 月 1 日 00:00:00 GMT 以来此 Date 对象表示的毫秒数。 + if (start.getTime() >= end.getTime()) { + return null; + } + long date = random(start.getTime(), end.getTime()); + return new Date(date); + } catch (Exception e) { + e.printStackTrace(); + } + return null; + } + + public static long random(long begin, long end) { + long rtn = begin + (long) (Math.random() * (end - begin)); + // 如果返回的是开始时间和结束时间,则递归调用本函数查找随机值 + if (rtn == begin || rtn == end) { + return random(begin, end); + } + return rtn; + } + }