From e43bd1d3087745329210f624e0b4d8db17a8fd0c Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 24 Oct 2022 11:46:02 +0800 Subject: [PATCH] update --- .../ccbevaluation/BigdataDealBatch.groovy | 6 ++- .../ccbevaluation/BigdataForsaleBatch.groovy | 18 +++++++ .../ccbevaluation/BigdataPoiBatch.groovy | 53 ++++++++++--------- .../ccbevaluation/CcbCommonUtils.groovy | 1 + .../DealAndThirdPriceDataProcess.groovy | 9 ++-- .../ccbevaluation/ExportOracleData.groovy | 1 + .../ccbevaluation/GuiDingPriceBatch.groovy | 4 +- .../ccbevaluation/ThirdPartyPriceBatch.groovy | 12 ++--- .../fangjia/script/gj/YouDaDataParser.groovy | 4 +- .../gj/paimai/BeiKeCrawlByChrome.groovy | 16 ++++-- .../gj/paimai/LianjiaCrawlByChrome.groovy | 2 +- .../script/gj/paimai/PaimaiDataSend.groovy | 8 +-- 12 files changed, 82 insertions(+), 52 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 89b9f7f35..3456607da 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -37,7 +37,7 @@ class BigdataDealBatch extends BaseBatch{ // def endDate = 202208 def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd")); - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"] public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017"; @@ -71,7 +71,9 @@ class BigdataDealBatch extends BaseBatch{ def rows = [] def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def index = 0 for(def city : citys){ + index ++ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def ppeDists = [:] @@ -87,7 +89,7 @@ class BigdataDealBatch extends BaseBatch{ def trades = getCityData(city, statMap, ppeDists) def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") rows.addAll(rets) - println city +" 成交共:"+rets.size() + println "【${index}】"+city +" 成交共:"+rets.size() def cityDatas = [] cityDatas << headers diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 95ec61b0d..573d65d6c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -68,6 +68,7 @@ class BigdataForsaleBatch extends BaseBatch{ public void process(Object args) { def index = 0 def citys = CcbCommonUtils.getRunningCitys(args) + statCity(citys) def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() for(def city : citys){ index++ @@ -82,6 +83,23 @@ class BigdataForsaleBatch extends BaseBatch{ } } + + def statCity(def citys){ + def datas = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\无房源城市.xlsx",0,1) + def houseCitys = [] + for(def d : datas){ + houseCitys << d['C'] + } + for(def city : citys){ + if(!houseCitys.contains(city)){ + def scity = CcbCommonUtils.specialCityMap.get(city) + if(!scity){ + println city + } + } + } + } + def exportSellCSV(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate){ def totalCount = new AtomicLong() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataPoiBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataPoiBatch.groovy index 39a26972b..08653a46a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataPoiBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataPoiBatch.groovy @@ -31,39 +31,40 @@ class BigdataPoiBatch extends BaseBatch{ @Override public void process(Object args) { def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() - def citys = CcbCommonUtils.citys; -// def path = "D:\\数据\\建行\\房产估值\\poi\\公交.xlsx" - def subwaydatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\地铁.xlsx", 0,1) - def busDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\公交.xlsx", 0,1) -// def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\医院.xlsx", 0,1) - def bankDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\银行.xlsx", 0,1) - def dealDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\丧葬服务.xlsx", 0,1) -// def schoolDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\学校.xlsx", 0,1) + def citys = CcbCommonUtils.citys_252 + + +// def dealDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\POI数据_丧葬设施_252.xlsx", 0,1) +// def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\POI数据_医院_聚类2_252.xlsx", 0,1) +// def subwaydatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\POI数据_地铁_252.xlsx", 0,1) +// def busDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\POI数据_公交_252.xlsx", 0,1) +// def bankDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\POI数据_银行_252.xlsx", 0,1) + + def schoolDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\POI数据_学校_252.xlsx", 0,1) // def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\八城市医院(二次审核).xlsx", 0,1) def rets = [] // def rows1 = createDatas(citys, cityCodeDataMap, hosptialDatas, "医院") // rets.addAll(rows1) - - def rows1 = createDatas(citys, cityCodeDataMap, subwaydatas, "地铁") - rets.addAll(rows1) - - def rows2 = createDatas(citys, cityCodeDataMap, busDatas, "公交") - rets.addAll(rows2) // -// def rows3 = createDatas(citys, cityCodeDataMap, hosptialDatas, "医院") +// def rows2 = createDatas(citys, cityCodeDataMap, busDatas, "公交") +// rets.addAll(rows2) +// +// def rows3 = createDatas(citys, cityCodeDataMap, bankDatas, "银行") // rets.addAll(rows3) - - def rows4 = createDatas(citys, cityCodeDataMap, bankDatas, "银行") - rets.addAll(rows4) - - def rows5 = createDatas(citys, cityCodeDataMap, dealDatas, "丧葬服务") - rets.addAll(rows5) +//// +// def rows4 = createDatas(citys, cityCodeDataMap, dealDatas, "丧葬服务") +// rets.addAll(rows4) // -// def rows6 = createDatas(citys, cityCodeDataMap, schoolDatas, "学校") -// rets.addAll(rows6) + def rows6 = createDatas(citys, cityCodeDataMap, schoolDatas, "学校") + rets.addAll(rows6) - def headers = ["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","性质","级别","来源公司"] + +// def rows5 = createDatas(citys, cityCodeDataMap, subwaydatas, "地铁") +// rets.addAll(rows5) +// + + def headers = []//["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","性质","级别","来源公司"] def fileName = "/home/goojia/ccb/BIGDATA_POI_GJ.dat" if(CommonDatautils.isWindow()){ @@ -260,9 +261,9 @@ class BigdataPoiBatch extends BaseBatch{ }else if(type =="丧葬服务"){ ret << "ob101" }else if(type =="学校"){ - if(d['F'].contains("中学")){ + if(d['B'].contains("中学")){ ret << "ed138" - }else if(d['F'].contains("小学")){ + }else if(d['B'].contains("小学")){ ret << "ed114" } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 62ce6f897..46aa81fac 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -417,6 +417,7 @@ class CcbCommonUtils { q.put("visible",0) q.put("cat", cat); q.put("city", city); +// q.put("name","云山尚城") // q.put("name_cat", 0); // q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"])) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy index e01954de1..a96478424 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy @@ -370,13 +370,12 @@ class DealPriceDataProcess { def deleteThirdPartyData(){ - def citys = CcbCommonUtils.citys - def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() - def month = 202208 + def citys = CcbCommonUtils.citys_92 + def month = 202209 def rows = [] def cityCodeDatas = CcbCommonUtils.findCityCodeValue() for(def city : citys) { - def rets = findPeGteThirdParty(city, 202209) + def rets = findPeGteThirdParty(city, 202210) def total = 0 println city +" "+rets.size() for(def m : rets){ @@ -388,7 +387,7 @@ class DealPriceDataProcess { } def findPeGteThirdParty(cityName, yearmonth){ - def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>30" + def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>25" def rets = [] sqlInstance236.eachRow(sql,{r-> def map = [:] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 0fbc2daa8..50e24d4fd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -171,6 +171,7 @@ class ExportOracleData { def exportCommunityStaging() { + def allRows = [] def sql = "select ID,COMMID,MASTER_COMMID,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_staging_gj"; sqlInstance.eachRow(sql, { r -> def data = [] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 936022f98..ba2d7db15 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -36,7 +36,7 @@ class GuiDingPriceBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = CcbCommonUtils.citys; + def citys = CcbCommonUtils.citys_252; def rows = [] for (def city : citys) { @@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2022-09-22", "2022-09-23") + def prices = CcbCommonUtils.findPersonPrice(city, "2021-09-01", "2022-10-12") if(prices){ def distMap = getDistrictMap(city) def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index c4a27815e..ec63b0189 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -22,10 +22,10 @@ import java.util.concurrent.ConcurrentHashMap @BatchOwner(Watcher.GaoJuan) class ThirdPartyPriceBatch extends BaseBatch { - def sDate= 202208//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); - def startDate = 202206 - def endDate = 202208 - def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); + def sDate= 202109//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def startDate = 202107 + def endDate = 202109 +// def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") @@ -42,7 +42,7 @@ class ThirdPartyPriceBatch extends BaseBatch { @Override public void process(Object args) { - def citys = CcbCommonUtils.citys; + def citys = CcbCommonUtils.citys_252; def rows = [] def index = 0 def weekend = DateUtils.getFirstDayOfWeek(new Date()) @@ -67,7 +67,7 @@ class ThirdPartyPriceBatch extends BaseBatch { } def header = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"] - FileUtils.batchCreateFile(rows, fileName, header, "utf-8", false); + FileUtils.batchCreateFile(rows, fileName, header, "utf-8", true); } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index 616012302..2aed5a7f1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -48,11 +48,11 @@ class YouDaDataParser { def final AtomicInteger count_ = new AtomicInteger(0) - static def statDate = "2022-08-01" + static def statDate = "2022-09-01" static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo -// task.esfParser("D:\\数据\\友达\\2022年9月\\查询1.DBF");//友达二手房存入mongo +// task.esfParser("D:\\数据\\友达\\2022年10月\\查询1.DBF");//友达二手房存入mongo // task.newhouseToSaveEs();//新房保存到es task.esfhouseToSaveEs();//二手房保存到es // task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index bb4f74fdf..b63201d66 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -49,7 +49,15 @@ public class BeiKeCrawlByChrome { WebDriver driver = new ChromeDriver() def citys = getAllCityUrls(driver) - citys = citys.reverse() +// citys = citys.reverse() +// def index = 0 +// for(def c : citys){ +// index++ +// if(c.city=="合肥"){ +// break +// } +// } +// citys = citys[index..citys.size()-1] def start = 0 // for(def i=0;i