From 592c8642333a38379932b59e8fc470d49da8488e Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 15 Mar 2021 22:20:41 +0800 Subject: [PATCH] update --- .../fangjia/script/gj/ccb/HouseExport.groovy | 83 ++++++-- .../script/gj/test/AnjukeCrawlToDB.groovy | 199 ++++++++++++++++++ 2 files changed, 264 insertions(+), 18 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/AnjukeCrawlToDB.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy index 3dbeebc2f..1ed9f2272 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy @@ -65,15 +65,15 @@ class HouseExport { def run(){ - def citys = ["北京", "哈尔滨", "重庆"] + def citys = [ "重庆", "北京"] for(def city : citys){ getRegionCodeByCity(city) def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias","finish_building_date","house_property","property_life","build_type","estate_type2","build_structure"]); - exportSellCSV(city, districts, 20170801, 20180901, 2017); - exportRentCSV(city, districts, 20170801, 20180901, 2017); + exportSellCSV(city, districts, 20170801, 20181001, 2017); + exportRentCSV(city, districts, 20170801, 20181001, 2017); - exportSellCSV(city, districts, 20200201, 20210201, 2020); - exportRentCSV(city, districts, 20200201, 20210201, 2020); + exportSellCSV(city, districts, 20200201, 20210301, 2020); + exportRentCSV(city, districts, 20200201, 20210301, 2020); } } @@ -87,11 +87,12 @@ class HouseExport { def retMap = new ConcurrentHashMap() for(def i=0;i=5 || seawater.room<1)){ - continue; + if(!seawater.room){ + seawater.room = null + } + + if(seawater.room){ + if(seawater.room>=5 || seawater.room<1){ + continue; + } } if(!seawater.hall){ @@ -540,15 +568,21 @@ class HouseExport { continue; } + def floor = parseFloor(seawater.floor_info) if(seawater.height && floor && floor.isNumber()){ if(floor.toInteger()>seawater.height){ floor = null seawater.height = null } - if(floor && floor.toInteger()<1){ - floor = null - } + + } + + if(!floor){ + seawater.floor = null + } + if(floor && floor.toInteger()<1){ + floor = null } def rowMap = [:]; @@ -557,11 +591,16 @@ class HouseExport { if(seawater.avg_price && seawater.avg_price>1000){ continue } + + if(seawater.avg_price && seawater.avg_price<10){ + continue + } + } rowMap.houseId = seawater.id rowMap.city = city - rowMap.region =seawater.region - rowMap.address = seawater.address + //rowMap.region =seawater.region + //rowMap.address = seawater.address rowMap.area = seawater.area @@ -698,6 +737,14 @@ class HouseExport { } + /* + * 生成一个用不重复的ID + */ +def GenNonDuplicateID(randomLength){ + return Number(Math.random().toString().substr(3,randomLength) + Date.now()).toString(36) +} +//GenNonDuplicateID()将生成 rfmipbs8ag0kgkcogc 类似的ID + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/AnjukeCrawlToDB.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/AnjukeCrawlToDB.groovy new file mode 100644 index 000000000..8c5009841 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/AnjukeCrawlToDB.groovy @@ -0,0 +1,199 @@ +package com.fangjia.script.gj.test + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import org.bson.types.ObjectId + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class AnjukeCrawlToDB { + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + + def totalCount = new AtomicLong() + + static main(args) { + def task = new AnjukeCrawlToDB(); + +// task.run() + + task.stat() + } + + def stat(){ + def districts = CommonDatautils.findAllDistricts("北京", "district", ["estate_charge","finish_building_date","build_area","household_count","car_site_count","cubage_percent","virescence_percent","estate_company","developer"]); + + def total = districts.size() + def estate_chargeCount = 0 + def build_areaCount = 0 + def household_countCount = 0 + def finish_building_dateCount = 0 + def car_site_countCount = 0 + def cubage_percentCount = 0 + def virescence_percentCount = 0 + def developerCount = 0 + def estate_companyCount = 0 + for(def dists : districts){ + if(dists.estate_charge){ + estate_chargeCount++ + } + // 总建面积 + if(dists.build_area){ + build_areaCount++ + } + // 总数 + if(dists.household_count){ + household_countCount++ + } + // 建造年代 + if(dists.finish_building_date){ + finish_building_dateCount++ + } + // 停车位 + if(dists.car_site_count){ + car_site_countCount++ + } + // 容积率 + if(dists.cubage_percent){ + cubage_percentCount++ + } + // 绿化率 + if(dists.virescence_percent){ + virescence_percentCount++ + } + // 开发商 + if(dists.developer){ + developerCount++ + } + // 物业公司 + if(dists.estate_company){ + estate_companyCount++ + } + } + + + def estate_charge_pre = DataHandleUtils.getDoubleRound(2, estate_chargeCount*100/total) + def build_area_pre = DataHandleUtils.getDoubleRound(2, build_areaCount*100/total) + def household_count_pre = DataHandleUtils.getDoubleRound(2, household_countCount*100/total) + def finish_building_date_pre = DataHandleUtils.getDoubleRound(2, finish_building_dateCount*100/total) + def car_site_pre = DataHandleUtils.getDoubleRound(2, car_site_countCount*100/total) + def cubage_percent_pre = DataHandleUtils.getDoubleRound(2, cubage_percentCount*100/total) + def virescence_percent_pre = DataHandleUtils.getDoubleRound(2, virescence_percentCount*100/total) + def developer_pre = DataHandleUtils.getDoubleRound(2, developerCount*100/total) + def estate_company_pre = DataHandleUtils.getDoubleRound(2, estate_companyCount*100/total) + + println finish_building_date_pre + println virescence_percent_pre + println cubage_percent_pre + + println build_area_pre + println estate_charge_pre + println developer_pre + println estate_company_pre +// println household_count_pre +// println car_site_pre + + + + + } + + + + + def run(){ + def datas = ExcelUtils.read("D:\\数据\\待抓取小区_安居客_detail.xlsx",0,1) + + + final CountDownLatch latch = new CountDownLatch(datas.size()) + for(def i=0;i0){ + udata.put("estate_charge",estateCharge) + } + } + // 总建面积 + if(!dists.build_area && data['L']){ + def buildArea = data['L'].toString().replace("m?","") + if(buildArea){ + udata.put("build_area",buildArea.toDouble().toInteger().toString()) + } + + } + // 总数 + if(!dists.household_count && data['M']){ + def householdCount= data['M'] + if(householdCount && householdCount.toString().isNumber()){ + udata.put("household_count",householdCount.toInteger()) + } + } + // 建造年代 + if(!dists.finish_building_date && data['N']){ + def year = data['N'].replace("年","") + if(year && year.isNumber()){ + udata.put("finish_building_date",year.toString()) + } + } + // 停车位 + if(!dists.car_site_count && data['O']){ + if(data['O'].isNumber()){ + udata.put("car_site_count", data['O'].toInteger()) + } + } + // 容积率 + if(!dists.cubage_percent && data['R']){ + def cubage_percent = data['R'] + if(cubage_percent && cubage_percent.isNumber() && cubage_percent.toDouble()<10){ + udata.put("cubage_percent", cubage_percent.toString()) + } + } + // 绿化率 + if(!dists.virescence_percent && data['Q']){ + def virescence_percent = data['Q'].toString().replace("%","") + if(virescence_percent && virescence_percent.isNumber()){ + udata.put("virescence_percent", virescence_percent.toString()) + } + } + // 开发商 + if(!dists.developer && data['R'] && !data['R'].contains("未知") && !data['R'].contains("无")){ + udata.put("developer", data['R']) + } + // 物业公司 + if(!dists.estate_company && data['S'] && !data['S'].contains("未知") && !data['S'].contains("无")){ + udata.put("estate_company",data['S']) + } + + if(udata){ + udata.put("m_date", new Date()) + println totalCount.get()+" "+data['A']+" "+udata + DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(data['A'])),new BasicDBObject('$set', udata)); + } + + } + } +}