城市行情新增字段统计

This commit is contained in:
2023-04-07 11:11:57 +08:00
parent 402fc0641a
commit f6d308359b
2 changed files with 524 additions and 20 deletions
@@ -28,28 +28,45 @@ class BigdataCitypriceBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
def curRunDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMM").toInteger();
def curRunDate = null
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
"postgres", "postgres", "org.postgresql.Driver")
def oracleSqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
public static def cityCodeMap = CcbCommonUtils.findCityCodeValue()
static main(args) {
runBatch(new BigdataCitypriceBatch(), args)
}
@Override
void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args)
def startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-1), "yyyyMMdd").toInteger();
def endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(new Date(),-1), "yyyyMMdd").toInteger();
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def curDate = new Date()
def citys = CcbCommonUtils.citys_all //CcbCommonUtils.getRunningCitys(args)
if(args){
def dateStr = args[0]
curDate = DateUtils.parseFromTightByPattern(dateStr,"yyyy-MM-dd")//2022-03-01
}
println "args 参数:"+args.toString() +" 城市数量:"+citys.size()+" curDate: "+ curDate
curRunDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(curDate,-1), "yyyyMM").toInteger();
def startDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(curDate,-1), "yyyyMMdd").toInteger();
def endDate = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(curDate,-1), "yyyyMMdd").toInteger();
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def dealStartDate = DateUtils.addMonth(DateUtils.parseFromTightByPattern(startDate.toString(), "yyyyMMdd"), -1)
def lastYearDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(curDate,-12), "yyyyMMdd").toInteger();
cityRun(citys, startDate, endDate, cityCodeDataMap)
regionRun(citys, startDate, endDate, cityCodeDataMap)
districtRun(citys, startDate, endDate, cityCodeDataMap)
districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeDataMap)
}
def cityRun(citys, startDate, endDate, cityCodeDataMap){
def cityRun(citys, startDate, endDate, dealStartDate, cityCodeDataMap){
def rets = []
def index = 0
for(def city : citys){
@@ -177,11 +194,18 @@ class BigdataCitypriceBatch extends BaseBatch{
println "end: "+new Date()
}
def districtRun(citys, startDate, endDate, cityCodeDataMap){
def districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeDataMap){
def rets = []
def allDistricts = []
def allPriceMap = [:]
def allRentMap = [:]
def allDealCountMap = [:]
def allCityRankMap = [:]
def allRegionRankMap = [:]
def allDistCityPreMap = [:]
def allDistRegionPreMap = [:]
def allLastMonthMap = [:]
def allLastYearMap = [:]
for(def city : citys) {
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias"]);
allDistricts.addAll(districts)
@@ -192,9 +216,36 @@ class BigdataCitypriceBatch extends BaseBatch{
def rentMap = CommonDatautils.findSqmetreRentByCityAndDate(city, endDate)
allRentMap = allRentMap + rentMap
def cityCode = cityCodeMap.get(city)
def dealCountMap = findDealDatesByCity(cityCode, dealStartDate, startDate)
allDealCountMap = allDealCountMap + dealCountMap
def datas = findCommunityPriceByCity(cityCode, startDate.toString().substring(0,6))
def cityRankMap = findCommunityCityRank(datas)
allCityRankMap = allCityRankMap + cityRankMap
def regionRankMap = findCommunityRegionRank(datas)
allRegionRankMap = allRegionRankMap + regionRankMap
def cityPrice = findCcbCityPrice(cityCode, startDate.toString().substring(0,6))
def regionPriceMap = findCcbRegionPrice(cityCode, startDate.toString().substring(0,6))
def cityPreMap = districtCityPrictRate(cityPrice, datas)
allDistCityPreMap = allDistCityPreMap + cityPreMap
def regionPreMap = districtRegionPriceRate(regionPriceMap, datas)
allDistRegionPreMap = allDistRegionPreMap + regionPreMap
def lastMonthMap = findDistQuotations(cityCode, dealStartDate.replace("-","").substring(0,6))
allLastMonthMap = allLastMonthMap + lastMonthMap
def lastYearMonthMap = findDistQuotations(cityCode, lastYearDate.toString().substring(0,6))
allLastYearMap = allLastYearMap + lastYearMonthMap
}
def cityLists = districtRunningByCity(allDistricts, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap)
def cityLists = districtRunningByCity(allDistricts, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, allDealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap)
if(cityLists){
rets.addAll(cityLists)
}
@@ -212,7 +263,7 @@ class BigdataCitypriceBatch extends BaseBatch{
def districtRunningByCity(districts, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap){
def districtRunningByCity(districts, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap){
def totalCount = new AtomicLong()
def distCount = districts.size()
final CountDownLatch latch = new CountDownLatch(distCount)
@@ -224,7 +275,7 @@ class BigdataCitypriceBatch extends BaseBatch{
void run() {
try {
totalCount.addAndGet(1)
def row = createDistDate(dist, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap)
def row = createDistDate(dist, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap,allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap)
if(row){
rets << row
println totalCount.get() +"/"+distCount + dist.city+" "+dist.region+" "+dist.name+" "+row[4]+" "+row[6]
@@ -271,7 +322,7 @@ class BigdataCitypriceBatch extends BaseBatch{
}
def createDistDate(dist, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap){
def createDistDate(dist, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap){
def sellDataType = 1
def rentDataType = 1
def sellMap = findHouses(dist.city, dist.region, "sell", startDate, endDate, dist.name, dist.alias, dist.estate_type2)
@@ -400,6 +451,155 @@ class BigdataCitypriceBatch extends BaseBatch{
row << 1
row << 'FJW'
//成交样本量,成交样本量同比, 成交样本量环比
if(dealCountMap){
def dealCount = dealCountMap.get(dist._id.toString())
row << dealCount?dealCount:''
def dealMarkup = ''
def dealYoy = ''
if(allLastMonthMap){
def dealMarkupMap = allLastMonthMap.get(dist._id.toString())
if(dealMarkupMap && dealMarkupMap.deal_count){
def markup = (dealCount-dealMarkupMap.deal_count)*100/dealMarkupMap.deal_count
dealMarkup = DataHandleUtils.getDoubleRound(2, markup)
}
}
if(allLastYearMap){
def dealYoyMap = allLastYearMap.get(dist._id.toString())
if(dealYoyMap && dealYoyMap.deal_count){
def markup = (dealCount-dealYoyMap.deal_count)*100/dealYoyMap.deal_count
dealYoy = DataHandleUtils.getDoubleRound(2, markup)
}
}
row << dealYoy
row << dealMarkup
}else{
row << ''
row << ''
row << ''
}
//出售平均单价的同比,环比
if(sell_avg_price){
def markup = ''
if(allLastMonthMap){
def dataMap = allLastMonthMap.get(dist._id.toString())
if(dataMap && dataMap.saleprice){
markup = (sell_avg_price - dataMap.saleprice)*100/dataMap.saleprice
markup = DataHandleUtils.getDoubleRound(2, markup)
}
}
def yoy = ''
if(allLastYearMap){
def dataMap = allLastYearMap.get(dist._id.toString())
if(dataMap && dataMap.saleprice){
yoy = (sell_avg_price - dataMap.saleprice)*100/dataMap.saleprice
yoy = DataHandleUtils.getDoubleRound(2, markup)
}
}
row << yoy
row << markup
}else{
row << ''
row << ''
}
//出售样本量同比,环比
if(sell_count){
def markup = ''
if(allLastMonthMap){
def dataMap = allLastMonthMap.get(dist._id.toString())
if(dataMap && dataMap.salecount){
markup = (sell_count - dataMap.salecount)*100/dataMap.salecount
markup = DataHandleUtils.getDoubleRound(2, markup)
}
}
def yoy = ''
if(allLastYearMap){
def dataMap = allLastYearMap.get(dist._id.toString())
if(dataMap && dataMap.salecount){
yoy = (sell_count - dataMap.salecount)*100/dataMap.salecount
yoy = DataHandleUtils.getDoubleRound(2, markup)
}
}
row << yoy
row << markup
}else{
row << ''
row << ''
}
//出租房源量同比,环比
if(rent_count){
def markup = ''
if(allLastMonthMap){
def dataMap = allLastMonthMap.get(dist._id.toString())
if(dataMap && dataMap.salecount && dataMap.rentcount>0){
markup = (rent_count - dataMap.rentcount)*100/dataMap.rentcount
markup = DataHandleUtils.getDoubleRound(2, markup)
}
}
def yoy = ''
if(allLastYearMap){
def dataMap = allLastYearMap.get(dist._id.toString())
if(dataMap && dataMap.salecount && dataMap.rentcount>0){
yoy = (rent_count - dataMap.rentcount)*100/dataMap.rentcount
yoy = DataHandleUtils.getDoubleRound(2, markup)
}
}
row << yoy
row << markup
}else{
row << ''
row << ''
}
//小区价格城市排名
if(allCityRankMap){
def cityRank = allCityRankMap.get(dist._id.toString())
if(cityRank){
row << cityRank
}else{
row << ''
}
}else{
row << ''
}
//小区价格区域排名
if(allRegionRankMap){
def regionRank = allRegionRankMap.get(dist._id.toString())
if(regionRank){
row << regionRank
}else{
row << ''
}
}else{
row << ''
}
//高于区县平均价比率
if(allDistRegionPreMap){
def regionRadio = allDistRegionPreMap.get(dist._id.toString())
if(regionRadio){
row << regionRadio
}else{
row << ''
}
}else{
row << ''
}
//高于城市平均价比率
if(allDistCityPreMap){
def cityRadio = allDistCityPreMap.get(dist._id.toString())
if(cityRadio){
row << cityRadio
}else{
row << ''
}
}else{
row << ''
}
return row
}
return null
@@ -723,6 +923,131 @@ class BigdataCitypriceBatch extends BaseBatch{
}
def findDealDatesByCity(cityCode, startDate, endDate){
def sql = "SELECT gujia_comm_id,count(1) from base.t0831_bigdata_deal_gj_h WHERE urbn_id='${cityCode}' and txn_dt>='${startDate}' and txn_dt<'${endDate}' GROUP BY gujia_comm_id";
def rMap = [:]
pgsqlInstance.eachRow(sql, { r ->
rMap.put(r.gujia_comm_id, r.count)
})
return rMap;
}
def findCommunityCityRank(datas){
def values = []
for(def m : datas){
values << m.avg_price
}
values = values.sort().reverse()
def resMap = [:]
def total = values.size()
for(def m : datas){
// def index = 0
// for(def v : values){
// index++
// if(m.avg_price == v){
// break;
// }
// }
def index = values.indexOf(m.avg_price)+1
resMap.put(m.comm_id, index+"|"+total)
}
return resMap
}
def findCommunityRegionRank(datas){
def regionMap = [:]
for(def m : datas){
def prices = regionMap.get(m.region)
if(!prices){
prices = []
regionMap.put(m.region, prices)
}
prices << m.avg_price
prices.sort().reverse()
}
def resMap = [:]
for(def m : datas){
def prices = regionMap.get(m.region)
def total = prices.size()
def index = prices.indexOf(m.avg_price)+1
// for(def v : prices){
// index++
// if(m.avg_price == v){
// break;
// }
// }
resMap.put(m.comm_id, index+"|"+total)
}
return resMap
}
def findCommunityPriceByCity(cityCode, month){
def sql = "select gujia_admndiv_nm,gujia_bds_id,gujia_bds_clc_avprc from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityCode}' and gujiabds_clc_yrmo_val=${month}";
def datas = []
pgsqlInstance.eachRow(sql, { r ->
datas << ["comm_id":r.gujia_bds_id, "avg_price":r.gujia_bds_clc_avprc, "region":r.gujia_admndiv_nm]
})
return datas;
}
def findCcbCityPrice(cityCode, month){
def sql = "SELECT gujia_bds_clc_avprc from t_app_p9pha_plda.avm_city_price_gj_h where urbn_id='${cityCode}' and gujiabds_clc_yrmo_val=${month}";
def avgPrice = null
pgsqlInstance.eachRow(sql, { r ->
avgPrice = r.gujia_bds_clc_avprc
})
return avgPrice;
}
def findCcbRegionPrice(cityCode, month){
def sql = "SELECT gxda_admndiv_nm,gujia_bds_clc_avprc from t_app_p9pha_plda.avm_district_price_gj_h where urbn_id='${cityCode}' and gujiabds_clc_yrmo_val=${month}";
def regionPriceMap = [:]
pgsqlInstance.eachRow(sql, { r ->
regionPriceMap.put(r.gxda_admndiv_nm, r.gujia_bds_clc_avprc)
})
return regionPriceMap;
}
def districtCityPrictRate(cityPrice, datas){
def distCityRateMap = [:]
if(cityPrice){
for(def m : datas){
def val = (m.avg_price-cityPrice)*100/cityPrice
distCityRateMap.put(m.comm_id, DataHandleUtils.getDoubleRound(4, val))
}
}
return distCityRateMap;
}
def districtRegionPriceRate(regionPriceMap, datas){
def distRegionRateMap = [:]
if(regionPriceMap){
for(def m : datas){
def regionAvgPrice = regionPriceMap.get(m.region)
def val = (m.avg_price-regionAvgPrice)*100/regionAvgPrice
distRegionRateMap.put(m.comm_id, DataHandleUtils.getDoubleRound(4, val))
}
}
return distRegionRateMap;
}
def findDistQuotations(cityCode, month){
def sql = "select commid, deal_count, saleprice,salecount,rentcount from bigdata_commprice_fjw where ymid=${month} and cityid=${cityCode}";
def markupMap = [:]
oracleSqlInstance.eachRow(sql, { r ->
markupMap.put(r.commid, ["deal_count":r.DEAL_COUNT, "saleprice":r.SALEPRICE,"salecount":r.SALECOUNT,"rentcount":r.RENTCOUNT])
})
return markupMap;
}
@@ -48,7 +48,9 @@ class AnjukeDistrictMatch {
// task.matchCommunityByCsv()
// task.updteBuildAreaFieldByCsv()
task.updadteAllAreaFieldByCsv()
// task.updadteAllAreaFieldByCsv()
// task.updatePlotRateFieldByCsv()
task.updateFinishDateFieldByCsv()
}
def updateCredit(){
@@ -201,7 +203,7 @@ class AnjukeDistrictMatch {
def matchCommunityByCsv(){
def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站绿化率 (匹配)-1.csv")
def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站容积率(匹配)-2.csv")
def map = [:]
for(def i=0;i<datas.size();i++) {
final def data = datas[i]
@@ -209,7 +211,7 @@ class AnjukeDistrictMatch {
map.put(row[0]+"#"+row[1]+"#"+row[2], row[24]+"#"+row[25]+"#"+row[26]+"#"+row[27]+"#"+row[28])
}
def datas1 = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站占地面积.csv")
def datas1 = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站竣工时间.csv")
def lastRets = []
for(def i=0;i<datas1.size();i++) {
final def data = datas1[i]
@@ -219,7 +221,9 @@ class AnjukeDistrictMatch {
def arrs = resultStr.split("#")
if(arrs){
row << arrs[0]
row << arrs[1]
if(arrs.size()>1){
row << arrs[1]
}
if(arrs.size()>2){
row << arrs[2]
}
@@ -233,7 +237,7 @@ class AnjukeDistrictMatch {
}
lastRets << row
}
batchCreateFile(lastRets, "D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站占地面积(匹配).csv",null,"GBK")
batchCreateFile(lastRets, "D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站竣工时间(匹配).csv",null,"GBK")
}
def updteGreenRateFieldByCsv(){
@@ -465,15 +469,190 @@ class AnjukeDistrictMatch {
}
}
def updatePlotRateFieldByCsv(){
def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站容积率(匹配)-2.csv")
for(def i=0;i<datas.size();i++) {
final def data = datas[i]
def row = data as List
if(row[22] && (row[22]=="名称匹配"||row[22]=="映射别名匹配" || (row[22].contains("接口") && row[26] && row[26].toString().isNumber() && row[26].toString().toDouble()>=0.9))){
def dist = CommonDatautils.findSeaweedById(row[23], ["cubage_percent"])
if(dist && !dist.cubage_percent){
def val = row[10]
if(!val){
val = row[11]
}
if(!val){
val = row[12]
}
if(!val){
val = row[19]
}
if(!val){
val = row[20]
}
if(!val){
val = row[21]
}
if(!val){
val = row[6]
}
if(!val){
val = row[8]
}
if(!val){
val = row[9]
}
if(!val){
val = row[13]
}
if(!val){
val = row[14]
}
if(!val){
val = row[17]
}
if(!val){
val = row[3]
}
if(!val){
val = row[4]
}
if(!val){
val = row[5]
}
if(!val){
val = row[15]
}
if(!val){
val = row[16]
}
if(!val){
val = row[18]
}
if(val){
if(val.toString().toDouble().toInteger()==val.toString().toDouble()){
val = val.toString().toDouble().toInteger()
}
def udata = new BasicDBObject()
udata.put("cubage_percent", val.toString())
udata.put("m_date",new Date())
def q = new BasicDBObject()
q.put("_id", new ObjectId(dist._id.toString()))
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
println row[23] +" "+val.toString()
}
}
}
}
}
// def run1(){
// def regionPointsMap = findRegionRange("重庆")
// def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, regionPointsMap.get(dist.region))
// }
def updateFinishDateFieldByCsv(){
def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\竣工时间.csv")
for(def i=0;i<datas.size();i++) {
final def data = datas[i]
def row = data as List
if(row[22] && (row[22]=="名称匹配"||row[22]=="映射别名匹配" || (row[22].contains("接口") && row[26] && row[26].toString().isNumber() && row[26].toString().toDouble()>=0.9))){
def dist = CommonDatautils.findSeaweedById(row[23], ["finish_building_date"])
if(dist && !dist.finish_building_date){
def val = row[10]
if(!val){
val = row[11]
}
if(!val){
val = row[12]
}
if(!val){
val = row[19]
}
if(!val){
val = row[20]
}
if(!val){
val = row[21]
}
if(!val){
val = row[6]
}
if(!val){
val = row[8]
}
if(!val){
val = row[9]
}
if(!val){
val = row[13]
}
if(!val){
val = row[14]
}
if(!val){
val = row[17]
}
if(!val){
val = row[3]
}
if(!val){
val = row[4]
}
if(!val){
val = row[5]
}
if(!val){
val = row[15]
}
if(!val){
val = row[16]
}
if(!val){
val = row[18]
}
if(val){
if(val.toString().toDouble().toInteger()==val.toString().toDouble()){
val = val.toString().toDouble().toInteger()
}
def udata = new BasicDBObject()
udata.put("finish_building_date", val.toString())
udata.put("m_date",new Date())
def q = new BasicDBObject()
q.put("_id", new ObjectId(dist._id.toString()))
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
println row[23] +" "+val.toString()
}
}
}
}
}
def runCsv(){
def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站绿化率 (3).csv")//第三方网站绿化率 (3).csv
def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\容积率未匹配.csv")//第三方网站绿化率 (3).csv
def results = batchMatch(datas)
batchCreateFile(results, "D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站绿化率 (匹配).csv",null,"GBK")
batchCreateFile(results, "D:\\数据\\建行\\二期\\第三方小区字段补充\\容积率未匹配-2.csv",null,"GBK")
}
@@ -488,7 +667,7 @@ class AnjukeDistrictMatch {
void run() {
try{
totalCount.addAndGet(1)
if (row[0] && row[1]) {
if (row[0] && row[1] && !row[22]) {
def city = row[0]
def region = row[1]
def name = row[2]