This commit is contained in:
2021-08-12 19:00:18 +08:00
parent ebe72544f5
commit e4e08ac004
3 changed files with 65 additions and 29 deletions
@@ -28,12 +28,13 @@ import java.util.regex.Pattern
class BigdataDealBatch extends BaseBatch{
def curDate = 20210601//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1)
def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"]
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
public static final DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
static main(args) {
runBatch(new BigdataDealBatch(), args);
@@ -41,6 +42,7 @@ class BigdataDealBatch extends BaseBatch{
@Override
public void process(Object args) {
def weekend = DateUtils.getFirstDayOfWeek(new Date())
def citys = CcbCommonUtils.citys;
def rows = []
def headers = ["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
@@ -48,13 +50,14 @@ class BigdataDealBatch extends BaseBatch{
for(def city : citys){
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def trades = getCityData(city)
def statMap = getDistrictStatByCity(city, weekend)
def trades = getCityData(city, statMap)
def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
rows.addAll(rets)
println city +" 成交共:"+rets.size()
if(city=="上海"){
def youDataTrade = getYouDaData()
def youDataTrade = getYouDaData(statMap)
def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
rows.addAll(youdaDatas)
println city +" "+youdaDatas.size()
@@ -386,7 +389,7 @@ class BigdataDealBatch extends BaseBatch{
return s;
}
def getCityData(def city){
def getCityData(def city, def statMap){
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","build_type","build_structure", "house_property","property_life"])
def tradesList = Collections.synchronizedList(new ArrayList());
@@ -405,11 +408,18 @@ class BigdataDealBatch extends BaseBatch{
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20210701", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
for(def i=0;i<dists.size();i++){
final def dist = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
// fixedThreadPool.execute(new Runnable() {
// void run() {
try {
totalCount.addAndGet(1)
def trades = CcbCommonUtils.findTradesDetail(dist, startDate, endDate)
def price = statMap.get(dist.region+"@"+dist.name)
def startPrice = null
def endPrice = null
if(price){
startPrice = (int)(price - price*0.5)
endPrice = (int)(price*0.5 + price)
}
def trades = CcbCommonUtils.findTradesDetail(dist, startDate, endDate, startPrice, endPrice)
if(trades){
tradesList.addAll(trades)
}
@@ -419,14 +429,14 @@ class BigdataDealBatch extends BaseBatch{
} finally {
latch.countDown()
}
};
})
// };
// })
}
latch.await()
return tradesList
}
def getYouDaData(){
def getYouDaData(statMap){
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
def q = new BasicDBObject()
q.put("contractTime", new BasicDBObject('$gte', startDate));
@@ -435,10 +445,33 @@ class BigdataDealBatch extends BaseBatch{
def rets = []
for(def obj : dbcursor){
if(obj.fjName && ["公寓","联列住宅","花园住宅","别墅","里弄房","新工房"].contains(obj.houseType)){
rets << obj;
def price = statMap.get(obj.fjRegion+"@"+obj.fjName)
if(price){
def startPrice = (int)(price - price*0.5)
def endPrice= (int)(price*0.5 + price)
if(obj.price>=startPrice && obj.price<=endPrice){
rets << obj;
}
}
}
}
return rets;
}
def getDistrictStatByCity(city, weekend){
def q = new BasicDBObject()
q.put("city", city)
q.put("weekend", weekend)
def dbcursor = DIST_SELL_STAT.find(q)
def map = [:]
for(def d : dbcursor){
map.put(d.region+"@"+d.district_name, d.avg_price)
}
return map
}
}
@@ -12,7 +12,8 @@ import com.mongodb.DBObject
class CcbCommonUtils {
public static def citys = ["上海","北京", "重庆","天津","成都","沈阳","大连","陵水","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭"]
public static def citys = ["扬州","上海","北京", "重庆","天津","成都","沈阳","大连","陵水","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭"]
// public static def citys = ["武汉"]
public static def provCityMap = ["河北":"石家庄" ,"山西":"太原" ,"陕西":"西安" ,"山东":"济南","河南":"郑州" ,"辽宁":"沈阳" ,"吉林":"长春" ,"黑龙江":"哈尔滨" ,"江苏":"南京" ,"浙江":"杭州" ,"安徽":"合肥","江西":"南昌" ,"福建":"福州" ,"湖北":"武汉","湖南":"长沙" ,"四川":"成都" ,"贵州":"贵阳" ,"云南":"昆明" ,"广东":"广州" ,"海南":"海口" ,"甘肃":"兰州" ,"青海":"西宁" ,"台湾":"台北","内蒙古":"呼和浩特" ,"新疆":"乌鲁木齐" ,"西藏":"拉萨","广西":"南宁","宁夏":"银川","香港":"香港","澳门":"澳门"]
@@ -59,7 +60,7 @@ class CcbCommonUtils {
return rets
}
public static def findTradesDetail(dist, startDate, endDate){
public static def findTradesDetail(dist, startDate, endDate, startPrice, endPrice){
def q = new BasicDBObject()
q.put("city", dist.city)
q.put("region", dist.region)
@@ -83,18 +84,20 @@ class CcbCommonUtils {
def rets = new HashSet()
def lists = []
for(def d : dbcursor){
def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
if(!rets.contains(key)){
rets << key
d.dist_id = dist._id.toString()
d.address = dist.address
d.master_name = dist.name
d.build_type = dist.build_type
d.build_structure = dist.build_structure
d.house_roperty = dist.house_roperty
d.property_life = dist.property_life
d.estate_type2 = dist.estate_type2
lists << d
if(startPrice && d.avg_price>=startPrice && d.avg_price<=endPrice){
def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
if(!rets.contains(key)){
rets << key
d.dist_id = dist._id.toString()
d.address = dist.address
d.master_name = dist.name
d.build_type = dist.build_type
d.build_structure = dist.build_structure
d.house_roperty = dist.house_roperty
d.property_life = dist.property_life
d.estate_type2 = dist.estate_type2
lists << d
}
}
}
return lists
@@ -18,7 +18,7 @@ import com.mongodb.DBCollection
@BatchOwner(Watcher.GaoJuan)
class ThirdPartyPriceBatch extends BaseBatch {
def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
@@ -73,7 +73,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
ret << d.city
ret << data.district_code
ret << data.district
ret << third.s_date
ret << sDate
if(third.zhugezhaofang_esf_price){
ret << third.zhugezhaofang_esf_price
@@ -169,7 +169,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
def findThirdPartyPrice(def city){
def q = new BasicDBObject()
q.put("city", city)
q.put("s_date", new BasicDBObject('$gte', curDate))
q.put("s_date", new BasicDBObject('$gte', sDate))
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
def map = [:]
for(def db : dbcursor){