From 7a52c6764c4204efbc9a2b79be88f46bd57242df Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 20 Jul 2023 16:09:17 +0800 Subject: [PATCH] update --- .../ccbevaluation/BigdataDealBatch.groovy | 104 +++++++++--------- .../ccbevaluation/GuiDingPriceBatch.groovy | 6 +- .../script/ccbevaluation/TradeStatLog.groovy | 10 +- .../com/fangjia/script/gj/FiledStrand.groovy | 19 +++- .../gj/district/OfficeFieldCleanBatch.groovy | 30 +++-- .../gj/paimai/BeiKeCrawlByChrome.groovy | 5 +- 6 files changed, 104 insertions(+), 70 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 13fcabfc3..667458d29 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -37,7 +37,7 @@ class BigdataDealBatch extends BaseBatch{ def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd")); - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"] public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017"; @@ -61,58 +61,58 @@ class BigdataDealBatch extends BaseBatch{ public void process(Object args) { //def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date()) // def citys = CcbCommonUtils.citys; -// def citys = CcbCommonUtils.getRunningCitys(args) + def citys = CcbCommonUtils.getRunningCitys(args) if(args && args.size()>=2){ curDate = args[1].toString().toInteger() } println "curDate:"+curDate - def allCitys = ["福州"] //CommonDatautils.findAllCity() - for(def city : allCitys){ -// if(!ccbCitys.contains(city)){ - addDealPriceMatch(city); -// } +// def allCitys = CommonDatautils.findAllCity() +// for(def city : allCitys){ +//// if(!ccbCitys.contains(city)){ +// addDealPriceMatch(city); +//// } +// } + + + def rows = [] + def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def index = 0 + for(def city : citys){ + index ++ + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + def ppeDists = [:] + + def statMap = [:]//getDistrictStatByCity(city, weekend) + if(city=="上海"){ + def youDataTrade = getYouDaData(statMap) + def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda") + rows.addAll(youdaDatas) + println city +" "+youdaDatas.size() + } + + def trades = getCityData(city, statMap, ppeDists) + if(trades){ + def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") + rows.addAll(rets) + println "【${index}】"+city +" 成交共:"+rets.size() + + def cityDatas = [] + cityDatas << headers + cityDatas.addAll(rows) + } + } + def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" + if(CommonDatautils.isWindow()){ + fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" + } -// def rows = [] -// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] -// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() -// def index = 0 -// for(def city : citys){ -// index ++ -// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) -// def ppeDists = [:] -// -// def statMap = [:]//getDistrictStatByCity(city, weekend) -// if(city=="上海"){ -// def youDataTrade = getYouDaData(statMap) -// def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda") -// rows.addAll(youdaDatas) -// println city +" "+youdaDatas.size() -// } -// -// def trades = getCityData(city, statMap, ppeDists) -// if(trades){ -// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") -// rows.addAll(rets) -// println "【${index}】"+city +" 成交共:"+rets.size() -// -// def cityDatas = [] -// cityDatas << headers -// cityDatas.addAll(rows) -// } -// -// } -// -// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" -// if(CommonDatautils.isWindow()){ -// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" -// } -// -// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); -// println "txt 生成 " + rows.size() -// -// saveToPgSql(rows) + FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); + println "txt 生成 " + rows.size() + + saveToPgSql(rows) } def createData(def rets, cityCodeDatas, distCodeMap, source){ @@ -652,9 +652,9 @@ class BigdataDealBatch extends BaseBatch{ for(def i=0;i=0.35 && estateCharege<=60){ return estateCharege; }else{ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/OfficeFieldCleanBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/OfficeFieldCleanBatch.groovy index 44e6641d4..b4ff6f8e2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/OfficeFieldCleanBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/OfficeFieldCleanBatch.groovy @@ -36,10 +36,10 @@ class OfficeFieldCleanBatch extends BaseBatch{ "virescence_percent":"greening_rate","cubage_percent":"plot_ratio","build_area":"build_area","all_area":"land_area" ,"finish_building_date":"finish_building_date","property_life":"property_life","standard_area":"standard_area","total_floor":"total_floor", "standard_height":"floor_height", "real_use_percent":"room_rate", "loop_line_info":"ring_location","car_site_count":"parking", - "parking_fee":"car_fee","elevators_count":"elevator_num","bay_area":"open_area"]//key:seaweed表字段 value:es表字段 + "parking_fee":"car_fee","elevators_count":"elevator_num","bay_area":"open_area","estate_type2":"property_type"]//key:seaweed表字段 value:es表字段 - //"安居客":"安居客app", - def sourceMap = ["房天下":"房天下","楼盘网":"楼盘网","办办网":"办办网","点点租":"点点租"] + // + def sourceMap = ["安居客":"安居客app","房天下":"房天下","楼盘网":"楼盘网","办办网":"办办网","点点租":"点点租"] def fieldMap = ["编号":"_id","城市":"city","区域":"region","名称":"name","别名":"alias","租金":"rent_price","板块":"block","物业公司":"estate_company","物业费":"estate_charge", "地址":"address","地址别名":"addr_alias","经度":"lng","纬度":"lat","类型":"estate_type2","建筑面积":"build_area","等级":"bed_kind", @@ -71,11 +71,11 @@ class OfficeFieldCleanBatch extends BaseBatch{ // addBlock() -// addFieldValue() + addFieldValue() // updateErrorField() - export() - test() +// export() +// test() // addOffice() } @@ -478,7 +478,12 @@ class OfficeFieldCleanBatch extends BaseBatch{ if(esVal){ def newVal = null if(seaweedfield == "estate_charge"){ - newVal = FiledStrand.processEstateCharge(c, FiledStrand.getNumber(esVal), dist.estate_type2) + if(dist.cat == "office"){ + newVal = FiledStrand.processEstateCharge(c, FiledStrand.getNumber(esVal), "office") + }else{ + newVal = FiledStrand.processEstateCharge(c, FiledStrand.getNumber(esVal), dist.estate_type2) + } + newVal = newVal.toString() } if(seaweedfield == "build_date" || seaweedfield == "live_date"){ @@ -532,7 +537,12 @@ class OfficeFieldCleanBatch extends BaseBatch{ } if(seaweedfield == "build_type"){ - newVal = FiledStrand.processBuildType(esVal, dist.estate_type2) + if(dist.cat == "office") { + newVal = FiledStrand.processBuildType(esVal, "office") + }else{ + newVal = FiledStrand.processBuildType(esVal, dist.estate_type2) + } + } if(seaweedfield == "land_property"){ @@ -570,6 +580,10 @@ class OfficeFieldCleanBatch extends BaseBatch{ newVal = esVal } + if(seaweedfield == "estate_type2"){ + newVal = FiledStrand.processEstateType(esVal) + } + if(newVal){ newValues.put(seaweedfield, newVal) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index 3df686d65..88b01f6c1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -296,7 +296,10 @@ public class BeiKeCrawlByChrome { def detailDiv1 = JQueryUtils.select(".introContent .base", detailDoc) def detailDiv2 = JQueryUtils.select(".introContent .transaction", detailDoc) - def liftStr = detailDiv1.select("li:contains(梯户比例)").text().replace("梯户比例","") + def liftStr = null; + if(detailDiv1){ + liftStr = detailDiv1.select("li:contains(梯户比例)").text().replace("梯户比例","") + } if(liftStr){ detail.lift = DataFormatUtils.parseLift(liftStr) detail.lift_household = DataFormatUtils.parseHousehold(liftStr)