diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy index 9a7d539e9..0d33825c8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy @@ -28,28 +28,45 @@ class BigdataCitypriceBatch extends BaseBatch{ public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80) - def curRunDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM").toInteger(); + def curRunDate = null def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified", "postgres", "postgres", "org.postgresql.Driver") + def oracleSqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + public static def cityCodeMap = CcbCommonUtils.findCityCodeValue() + static main(args) { runBatch(new BigdataCitypriceBatch(), args) } @Override void process(Object args) { - def citys = CcbCommonUtils.getRunningCitys(args) - def startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMMdd").toInteger(); - def endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(new Date(),-1), "yyyyMMdd").toInteger(); - def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def curDate = new Date() + def citys = CcbCommonUtils.citys_all //CcbCommonUtils.getRunningCitys(args) + if(args){ + def dateStr = args[0] + curDate = DateUtils.parseFromTightByPattern(dateStr,"yyyy-MM-dd")//2022-03-01 + } + + println "args 参数:"+args.toString() +" 城市数量:"+citys.size()+" curDate: "+ curDate + + curRunDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(curDate,-1), "yyyyMM").toInteger(); + def startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(curDate,-1), "yyyyMMdd").toInteger(); + def endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(curDate,-1), "yyyyMMdd").toInteger(); + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def dealStartDate = DateUtils.addMonth(DateUtils.parseFromTightByPattern(startDate.toString(), "yyyyMMdd"), -1) + + def lastYearDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(curDate,-12), "yyyyMMdd").toInteger(); cityRun(citys, startDate, endDate, cityCodeDataMap) regionRun(citys, startDate, endDate, cityCodeDataMap) - districtRun(citys, startDate, endDate, cityCodeDataMap) + districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeDataMap) } - def cityRun(citys, startDate, endDate, cityCodeDataMap){ + def cityRun(citys, startDate, endDate, dealStartDate, cityCodeDataMap){ def rets = [] def index = 0 for(def city : citys){ @@ -177,11 +194,18 @@ class BigdataCitypriceBatch extends BaseBatch{ println "end: "+new Date() } - def districtRun(citys, startDate, endDate, cityCodeDataMap){ + def districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeDataMap){ def rets = [] def allDistricts = [] def allPriceMap = [:] def allRentMap = [:] + def allDealCountMap = [:] + def allCityRankMap = [:] + def allRegionRankMap = [:] + def allDistCityPreMap = [:] + def allDistRegionPreMap = [:] + def allLastMonthMap = [:] + def allLastYearMap = [:] for(def city : citys) { def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias"]); allDistricts.addAll(districts) @@ -192,9 +216,36 @@ class BigdataCitypriceBatch extends BaseBatch{ def rentMap = CommonDatautils.findSqmetreRentByCityAndDate(city, endDate) allRentMap = allRentMap + rentMap + def cityCode = cityCodeMap.get(city) + def dealCountMap = findDealDatesByCity(cityCode, dealStartDate, startDate) + allDealCountMap = allDealCountMap + dealCountMap + + + def datas = findCommunityPriceByCity(cityCode, startDate.toString().substring(0,6)) + + def cityRankMap = findCommunityCityRank(datas) + allCityRankMap = allCityRankMap + cityRankMap + + def regionRankMap = findCommunityRegionRank(datas) + allRegionRankMap = allRegionRankMap + regionRankMap + + def cityPrice = findCcbCityPrice(cityCode, startDate.toString().substring(0,6)) + def regionPriceMap = findCcbRegionPrice(cityCode, startDate.toString().substring(0,6)) + + def cityPreMap = districtCityPrictRate(cityPrice, datas) + allDistCityPreMap = allDistCityPreMap + cityPreMap + + def regionPreMap = districtRegionPriceRate(regionPriceMap, datas) + allDistRegionPreMap = allDistRegionPreMap + regionPreMap + + def lastMonthMap = findDistQuotations(cityCode, dealStartDate.replace("-","").substring(0,6)) + allLastMonthMap = allLastMonthMap + lastMonthMap + + def lastYearMonthMap = findDistQuotations(cityCode, lastYearDate.toString().substring(0,6)) + allLastYearMap = allLastYearMap + lastYearMonthMap } - def cityLists = districtRunningByCity(allDistricts, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap) + def cityLists = districtRunningByCity(allDistricts, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, allDealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap) if(cityLists){ rets.addAll(cityLists) } @@ -212,7 +263,7 @@ class BigdataCitypriceBatch extends BaseBatch{ - def districtRunningByCity(districts, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap){ + def districtRunningByCity(districts, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap){ def totalCount = new AtomicLong() def distCount = districts.size() final CountDownLatch latch = new CountDownLatch(distCount) @@ -224,7 +275,7 @@ class BigdataCitypriceBatch extends BaseBatch{ void run() { try { totalCount.addAndGet(1) - def row = createDistDate(dist, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap) + def row = createDistDate(dist, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap,allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap) if(row){ rets << row println totalCount.get() +"/"+distCount + dist.city+" "+dist.region+" "+dist.name+" "+row[4]+" "+row[6] @@ -271,7 +322,7 @@ class BigdataCitypriceBatch extends BaseBatch{ } - def createDistDate(dist, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap){ + def createDistDate(dist, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap){ def sellDataType = 1 def rentDataType = 1 def sellMap = findHouses(dist.city, dist.region, "sell", startDate, endDate, dist.name, dist.alias, dist.estate_type2) @@ -400,6 +451,155 @@ class BigdataCitypriceBatch extends BaseBatch{ row << 1 row << 'FJW' + + //成交样本量,成交样本量同比, 成交样本量环比 + if(dealCountMap){ + def dealCount = dealCountMap.get(dist._id.toString()) + row << dealCount?dealCount:'' + def dealMarkup = '' + def dealYoy = '' + if(allLastMonthMap){ + def dealMarkupMap = allLastMonthMap.get(dist._id.toString()) + if(dealMarkupMap && dealMarkupMap.deal_count){ + def markup = (dealCount-dealMarkupMap.deal_count)*100/dealMarkupMap.deal_count + dealMarkup = DataHandleUtils.getDoubleRound(2, markup) + } + } + if(allLastYearMap){ + def dealYoyMap = allLastYearMap.get(dist._id.toString()) + if(dealYoyMap && dealYoyMap.deal_count){ + def markup = (dealCount-dealYoyMap.deal_count)*100/dealYoyMap.deal_count + dealYoy = DataHandleUtils.getDoubleRound(2, markup) + } + } + row << dealYoy + row << dealMarkup + + }else{ + row << '' + row << '' + row << '' + } + + //出售平均单价的同比,环比 + if(sell_avg_price){ + def markup = '' + if(allLastMonthMap){ + def dataMap = allLastMonthMap.get(dist._id.toString()) + if(dataMap && dataMap.saleprice){ + markup = (sell_avg_price - dataMap.saleprice)*100/dataMap.saleprice + markup = DataHandleUtils.getDoubleRound(2, markup) + } + } + def yoy = '' + if(allLastYearMap){ + def dataMap = allLastYearMap.get(dist._id.toString()) + if(dataMap && dataMap.saleprice){ + yoy = (sell_avg_price - dataMap.saleprice)*100/dataMap.saleprice + yoy = DataHandleUtils.getDoubleRound(2, markup) + } + } + row << yoy + row << markup + }else{ + row << '' + row << '' + } + + //出售样本量同比,环比 + if(sell_count){ + def markup = '' + if(allLastMonthMap){ + def dataMap = allLastMonthMap.get(dist._id.toString()) + if(dataMap && dataMap.salecount){ + markup = (sell_count - dataMap.salecount)*100/dataMap.salecount + markup = DataHandleUtils.getDoubleRound(2, markup) + } + } + def yoy = '' + if(allLastYearMap){ + def dataMap = allLastYearMap.get(dist._id.toString()) + if(dataMap && dataMap.salecount){ + yoy = (sell_count - dataMap.salecount)*100/dataMap.salecount + yoy = DataHandleUtils.getDoubleRound(2, markup) + } + } + row << yoy + row << markup + }else{ + row << '' + row << '' + } + + //出租房源量同比,环比 + if(rent_count){ + def markup = '' + if(allLastMonthMap){ + def dataMap = allLastMonthMap.get(dist._id.toString()) + if(dataMap && dataMap.salecount && dataMap.rentcount>0){ + markup = (rent_count - dataMap.rentcount)*100/dataMap.rentcount + markup = DataHandleUtils.getDoubleRound(2, markup) + } + } + def yoy = '' + if(allLastYearMap){ + def dataMap = allLastYearMap.get(dist._id.toString()) + if(dataMap && dataMap.salecount && dataMap.rentcount>0){ + yoy = (rent_count - dataMap.rentcount)*100/dataMap.rentcount + yoy = DataHandleUtils.getDoubleRound(2, markup) + } + } + row << yoy + row << markup + }else{ + row << '' + row << '' + } + //小区价格城市排名 + if(allCityRankMap){ + def cityRank = allCityRankMap.get(dist._id.toString()) + if(cityRank){ + row << cityRank + }else{ + row << '' + } + }else{ + row << '' + } + //小区价格区域排名 + if(allRegionRankMap){ + def regionRank = allRegionRankMap.get(dist._id.toString()) + if(regionRank){ + row << regionRank + }else{ + row << '' + } + }else{ + row << '' + } + //高于区县平均价比率 + if(allDistRegionPreMap){ + def regionRadio = allDistRegionPreMap.get(dist._id.toString()) + if(regionRadio){ + row << regionRadio + }else{ + row << '' + } + }else{ + row << '' + } + + //高于城市平均价比率 + if(allDistCityPreMap){ + def cityRadio = allDistCityPreMap.get(dist._id.toString()) + if(cityRadio){ + row << cityRadio + }else{ + row << '' + } + }else{ + row << '' + } return row } return null @@ -723,6 +923,131 @@ class BigdataCitypriceBatch extends BaseBatch{ } + def findDealDatesByCity(cityCode, startDate, endDate){ + def sql = "SELECT gujia_comm_id,count(1) from base.t0831_bigdata_deal_gj_h WHERE urbn_id='${cityCode}' and txn_dt>='${startDate}' and txn_dt<'${endDate}' GROUP BY gujia_comm_id"; + def rMap = [:] + pgsqlInstance.eachRow(sql, { r -> + rMap.put(r.gujia_comm_id, r.count) + }) + return rMap; + } + + + def findCommunityCityRank(datas){ + def values = [] + for(def m : datas){ + values << m.avg_price + } + values = values.sort().reverse() + + def resMap = [:] + def total = values.size() + for(def m : datas){ +// def index = 0 +// for(def v : values){ +// index++ +// if(m.avg_price == v){ +// break; +// } +// } + def index = values.indexOf(m.avg_price)+1 + resMap.put(m.comm_id, index+"|"+total) + } + return resMap + } + + def findCommunityRegionRank(datas){ + + def regionMap = [:] + for(def m : datas){ + def prices = regionMap.get(m.region) + if(!prices){ + prices = [] + regionMap.put(m.region, prices) + } + prices << m.avg_price + prices.sort().reverse() + } + + + def resMap = [:] + for(def m : datas){ + def prices = regionMap.get(m.region) + def total = prices.size() + def index = prices.indexOf(m.avg_price)+1 +// for(def v : prices){ +// index++ +// if(m.avg_price == v){ +// break; +// } +// } + resMap.put(m.comm_id, index+"|"+total) + } + return resMap + } + + def findCommunityPriceByCity(cityCode, month){ + def sql = "select gujia_admndiv_nm,gujia_bds_id,gujia_bds_clc_avprc from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityCode}' and gujiabds_clc_yrmo_val=${month}"; + def datas = [] + pgsqlInstance.eachRow(sql, { r -> + datas << ["comm_id":r.gujia_bds_id, "avg_price":r.gujia_bds_clc_avprc, "region":r.gujia_admndiv_nm] + }) + return datas; + } + + def findCcbCityPrice(cityCode, month){ + def sql = "SELECT gujia_bds_clc_avprc from t_app_p9pha_plda.avm_city_price_gj_h where urbn_id='${cityCode}' and gujiabds_clc_yrmo_val=${month}"; + def avgPrice = null + pgsqlInstance.eachRow(sql, { r -> + avgPrice = r.gujia_bds_clc_avprc + }) + return avgPrice; + } + + + + def findCcbRegionPrice(cityCode, month){ + def sql = "SELECT gxda_admndiv_nm,gujia_bds_clc_avprc from t_app_p9pha_plda.avm_district_price_gj_h where urbn_id='${cityCode}' and gujiabds_clc_yrmo_val=${month}"; + def regionPriceMap = [:] + pgsqlInstance.eachRow(sql, { r -> + regionPriceMap.put(r.gxda_admndiv_nm, r.gujia_bds_clc_avprc) + }) + return regionPriceMap; + } + + + def districtCityPrictRate(cityPrice, datas){ + def distCityRateMap = [:] + if(cityPrice){ + for(def m : datas){ + def val = (m.avg_price-cityPrice)*100/cityPrice + distCityRateMap.put(m.comm_id, DataHandleUtils.getDoubleRound(4, val)) + } + } + return distCityRateMap; + } + + + def districtRegionPriceRate(regionPriceMap, datas){ + def distRegionRateMap = [:] + if(regionPriceMap){ + for(def m : datas){ + def regionAvgPrice = regionPriceMap.get(m.region) + def val = (m.avg_price-regionAvgPrice)*100/regionAvgPrice + distRegionRateMap.put(m.comm_id, DataHandleUtils.getDoubleRound(4, val)) + } + } + return distRegionRateMap; + } + + def findDistQuotations(cityCode, month){ + def sql = "select commid, deal_count, saleprice,salecount,rentcount from bigdata_commprice_fjw where ymid=${month} and cityid=${cityCode}"; + def markupMap = [:] + oracleSqlInstance.eachRow(sql, { r -> + markupMap.put(r.commid, ["deal_count":r.DEAL_COUNT, "saleprice":r.SALEPRICE,"salecount":r.SALECOUNT,"rentcount":r.RENTCOUNT]) + }) + return markupMap; + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy index 8ba85525b..b28cc2770 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy @@ -48,7 +48,9 @@ class AnjukeDistrictMatch { // task.matchCommunityByCsv() // task.updteBuildAreaFieldByCsv() - task.updadteAllAreaFieldByCsv() +// task.updadteAllAreaFieldByCsv() +// task.updatePlotRateFieldByCsv() + task.updateFinishDateFieldByCsv() } def updateCredit(){ @@ -201,7 +203,7 @@ class AnjukeDistrictMatch { def matchCommunityByCsv(){ - def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站绿化率 (匹配)-1.csv") + def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站容积率(匹配)-2.csv") def map = [:] for(def i=0;i1){ + row << arrs[1] + } if(arrs.size()>2){ row << arrs[2] } @@ -233,7 +237,7 @@ class AnjukeDistrictMatch { } lastRets << row } - batchCreateFile(lastRets, "D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站占地面积(匹配).csv",null,"GBK") + batchCreateFile(lastRets, "D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站竣工时间(匹配).csv",null,"GBK") } def updteGreenRateFieldByCsv(){ @@ -465,15 +469,190 @@ class AnjukeDistrictMatch { } } + + def updatePlotRateFieldByCsv(){ + def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站容积率(匹配)-2.csv") + for(def i=0;i=0.9))){ + def dist = CommonDatautils.findSeaweedById(row[23], ["cubage_percent"]) + if(dist && !dist.cubage_percent){ + def val = row[10] + + if(!val){ + val = row[11] + } + if(!val){ + val = row[12] + } + + if(!val){ + val = row[19] + } + if(!val){ + val = row[20] + } + if(!val){ + val = row[21] + } + + if(!val){ + val = row[6] + } + if(!val){ + val = row[8] + } + + if(!val){ + val = row[9] + } + if(!val){ + val = row[13] + } + if(!val){ + val = row[14] + } + + if(!val){ + val = row[17] + } + + if(!val){ + val = row[3] + } + if(!val){ + val = row[4] + } + if(!val){ + val = row[5] + } + + if(!val){ + val = row[15] + } + if(!val){ + val = row[16] + } + + if(!val){ + val = row[18] + } + + if(val){ + if(val.toString().toDouble().toInteger()==val.toString().toDouble()){ + val = val.toString().toDouble().toInteger() + } + def udata = new BasicDBObject() + udata.put("cubage_percent", val.toString()) + udata.put("m_date",new Date()) + + def q = new BasicDBObject() + q.put("_id", new ObjectId(dist._id.toString())) + DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) + println row[23] +" "+val.toString() + } + } + } + } + } + // def run1(){ // def regionPointsMap = findRegionRange("重庆") // def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, regionPointsMap.get(dist.region)) // } + def updateFinishDateFieldByCsv(){ + def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\竣工时间.csv") + for(def i=0;i=0.9))){ + def dist = CommonDatautils.findSeaweedById(row[23], ["finish_building_date"]) + if(dist && !dist.finish_building_date){ + def val = row[10] + + if(!val){ + val = row[11] + } + if(!val){ + val = row[12] + } + + if(!val){ + val = row[19] + } + if(!val){ + val = row[20] + } + if(!val){ + val = row[21] + } + + if(!val){ + val = row[6] + } + if(!val){ + val = row[8] + } + + if(!val){ + val = row[9] + } + if(!val){ + val = row[13] + } + if(!val){ + val = row[14] + } + + if(!val){ + val = row[17] + } + + if(!val){ + val = row[3] + } + if(!val){ + val = row[4] + } + if(!val){ + val = row[5] + } + + if(!val){ + val = row[15] + } + if(!val){ + val = row[16] + } + + if(!val){ + val = row[18] + } + + if(val){ + if(val.toString().toDouble().toInteger()==val.toString().toDouble()){ + val = val.toString().toDouble().toInteger() + } + def udata = new BasicDBObject() + udata.put("finish_building_date", val.toString()) + udata.put("m_date",new Date()) + + def q = new BasicDBObject() + q.put("_id", new ObjectId(dist._id.toString())) + DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) + println row[23] +" "+val.toString() + } + } + } + } + } + def runCsv(){ - def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站绿化率 (3).csv")//第三方网站绿化率 (3).csv + def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\容积率未匹配.csv")//第三方网站绿化率 (3).csv def results = batchMatch(datas) - batchCreateFile(results, "D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站绿化率 (匹配).csv",null,"GBK") + batchCreateFile(results, "D:\\数据\\建行\\二期\\第三方小区字段补充\\容积率未匹配-2.csv",null,"GBK") } @@ -488,7 +667,7 @@ class AnjukeDistrictMatch { void run() { try{ totalCount.addAndGet(1) - if (row[0] && row[1]) { + if (row[0] && row[1] && !row[22]) { def city = row[0] def region = row[1] def name = row[2]