This commit is contained in:
2021-10-13 17:25:04 +08:00
parent e64fa84ab8
commit a68ba35360
@@ -29,7 +29,7 @@ import java.util.regex.Pattern
@BatchOwner(Watcher.GaoJuan) @BatchOwner(Watcher.GaoJuan)
class BigdataDealBatch extends BaseBatch{ class BigdataDealBatch extends BaseBatch{
def curDate = 20210401//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd")); def curDate = 20210701//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"] def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"]
@@ -48,7 +48,7 @@ class BigdataDealBatch extends BaseBatch{
@Override @Override
public void process(Object args) { public void process(Object args) {
def weekend = DateUtils.getFirstDayOfWeek(new Date()) def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
def citys = CcbCommonUtils.citys; def citys = CcbCommonUtils.citys;
println citys.unique().size() println citys.unique().size()
def rows = [] def rows = []
@@ -58,21 +58,25 @@ class BigdataDealBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def ppeDists = [:] def ppeDists = [:]
if(["贵阳","佛山","杭州","沈阳","天津","中山","徐州","绍兴","南通","哈尔滨","惠州","长沙","青岛","长春","烟台","温州","太原","台州","石家庄","宁波","常州","大连","嘉兴","金华","昆明"].contains(city)){ // if(["贵阳","佛山","杭州","沈阳","天津","中山","徐州","绍兴","南通","哈尔滨","惠州","长沙","青岛","长春","烟台","温州","太原","台州","石家庄","宁波","常州","大连","嘉兴","金华","昆明"].contains(city)){
ppeDists = getPpeGTE15Datas(city) // ppeDists = getPpeGTE15Datas(city)
} // }
def statMap = getDistrictStatByCity(city, weekend)
def trades = getCityData(city, statMap, ppeDists)
def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
rows.addAll(rets)
println city +" 成交共:"+rets.size()
def statMap = [:]//getDistrictStatByCity(city, weekend)
if(city=="上海"){ if(city=="上海"){
def youDataTrade = getYouDaData(statMap) def youDataTrade = getYouDaData(statMap)
def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda") def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
rows.addAll(youdaDatas) rows.addAll(youdaDatas)
println city +" "+youdaDatas.size() println city +" "+youdaDatas.size()
} }
def trades = getCityData(city, statMap, ppeDists)
def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
rows.addAll(rets)
println city +" 成交共:"+rets.size()
// FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal_${city}_1.txt",headers, "utf-8", false); // FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal_${city}_1.txt",headers, "utf-8", false);
def cityDatas = [] def cityDatas = []
cityDatas << headers cityDatas << headers
@@ -411,33 +415,33 @@ class BigdataDealBatch extends BaseBatch{
def startDate = null; def startDate = null;
// if(citys.contains(city)){ // if(citys.contains(city)){
startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210401", "yyyyMMdd")) startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
// }else{ // }else{
// startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) // startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
// } // }
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20210801", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date()) def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
for(def i=0;i<dists.size();i++){ for(def i=0;i<dists.size();i++){
final def dist = dists[i] final def dist = dists[i]
fixedThreadPool.execute(new Runnable() { fixedThreadPool.execute(new Runnable() {
void run() { void run() {
try { try {
totalCount.addAndGet(1) totalCount.addAndGet(1)
def price = statMap.get(dist.region+"@"+dist.name) // def price = statMap.get(dist.region+"@"+dist.name)
def startPrice = null def startPrice = null
def endPrice = null def endPrice = null
if(price){ // if(price){
if(dist.city=="上海"){ // if(dist.city=="上海"){
startPrice = (int)(price - price*0.1) // startPrice = (int)(price - price*0.1)
endPrice = (int)(price*0.5 + price) // endPrice = (int)(price*0.5 + price)
}else if(dist.city=="杭州"){ // }else if(dist.city=="杭州"){
startPrice = (int)(price - price*0.15) // startPrice = (int)(price - price*0.15)
endPrice = (int)(price*0.15 + price) // endPrice = (int)(price*0.15 + price)
}else{ // }else{
startPrice = (int)(price - price*0.5) // startPrice = (int)(price - price*0.5)
endPrice = (int)(price*0.5 + price) // endPrice = (int)(price*0.5 + price)
} // }
} // }
def trades = CcbCommonUtils.findTradesDetail(dist, startDate, endDate, startPrice, endPrice) def trades = CcbCommonUtils.findTradesDetail(dist, startDate, endDate, startPrice, endPrice)
if(!trades && dist.address){ if(!trades && dist.address){
def flag = dist.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$') def flag = dist.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
@@ -447,12 +451,12 @@ class BigdataDealBatch extends BaseBatch{
} }
if(trades){ if(trades){
def unitPrice= ppeDistsMap.get(dist._id.toString()) // def unitPrice= ppeDistsMap.get(dist._id.toString())
if(unitPrice){ // if(unitPrice){
for(def t : trades){ // for(def t : trades){
t.avg_price = (int)(t.avg_price*0.03+t.avg_price) // t.avg_price = (int)(t.avg_price*0.03+t.avg_price)
} // }
} // }
tradesList.addAll(trades) tradesList.addAll(trades)
} }
println totalCount.get() +" "+dist.city+" "+dist.region+" "+dist.name+" "+trades.size()+" "+new Date().toLocaleString() println totalCount.get() +" "+dist.city+" "+dist.region+" "+dist.name+" "+trades.size()+" "+new Date().toLocaleString()
@@ -470,14 +474,18 @@ class BigdataDealBatch extends BaseBatch{
def getYouDaData(statMap){ def getYouDaData(statMap){
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
def q = new BasicDBObject() def q = new BasicDBObject()
q.put("contractTime", new BasicDBObject('$gte', startDate)); q.put("contractTime", new BasicDBObject('$gte', startDate).append('$lte', endDate));
q.put("area", new BasicDBObject('$gte', 20)) q.put("area", new BasicDBObject('$gte', 20))
// q.put("premisesName" , "呼玛二村")
def dbcursor = youda_esfhouse_coll.find(q).batchSize(2000) def dbcursor = youda_esfhouse_coll.find(q).batchSize(2000)
def rets = [] def rets = []
def index = 0
for(def obj : dbcursor){ for(def obj : dbcursor){
if(obj.fjName && ["公寓","联列住宅","花园住宅","别墅","里弄房","新工房"].contains(obj.houseType)){
if(obj.fjName && ["公寓","联列住宅","花园住宅","别墅","里弄房","新工房","其它"].contains(obj.houseType)){
def price = statMap.get(obj.fjRegion+"@"+obj.fjName) def price = statMap.get(obj.fjRegion+"@"+obj.fjName)
if(price){ if(price){
@@ -492,10 +500,11 @@ class BigdataDealBatch extends BaseBatch{
} }
if(obj.price>=startPrice && obj.price<=endPrice){ if(obj.price>=startPrice && obj.price<=endPrice){
rets << obj; rets << obj;
}else{
index++
println index + " "+obj
} }
} }
} }
} }
return rets; return rets;