This commit is contained in:
2021-09-10 11:01:36 +08:00
parent 5a1844b926
commit 4b3dee5308
11 changed files with 485 additions and 99 deletions
@@ -27,6 +27,8 @@ class CcbCommonUtils {
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
public static final DBCollection DISTRICT_GUIDE_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "district_guide_price", "mongo.db.fangjia.com", 28017)
public static def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42"]
public static DSCollection dataStoreConnection = null;
@@ -75,6 +77,17 @@ class CcbCommonUtils {
return rets
}
public static def findGuidePrice(city){
def q = new BasicDBObject()
q.put("city", city)
def dbcursor = DISTRICT_GUIDE_PRICE_COLL.find(q)
def rets = []
for(def d : dbcursor){
rets << d
}
return rets
}
public static def findTradesDetail(dist, startDate, endDate, startPrice, endPrice){
def q = new BasicDBObject()
q.put("city", dist.city)
@@ -0,0 +1,85 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
@BatchOwner(Watcher.GaoJuan)
class DistrictGuidePriceBatch extends BaseBatch{
static main(args) {
runBatch(new DistrictGuidePriceBatch(), args);
}
@Override
public void process(Object args) {
def citys = ["深圳"]//CcbCommonUtils.citys;
def rows = []
for (def city : citys) {
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findGuidePrice(city)
if(prices){
def distMap = getDistrictMap(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
rows.addAll(rets)
println city + " " + rets.size()
}
}
// def headers = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导价格","类型","备注","创建时间","更新时间","来源公司"]
// FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\district_guied_price.dat", headers, "utf-8", false);
}
def getDistrictMap(def city){
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]);
def distMap = [:]
for(def d : districts){
distMap.put(d.region+"##"+d.name, d)
}
return distMap;
}
def createCityDatas(cityCodeDatas, distCodeMap, distMap, prices){
def rets = []
if(prices){
for(def ps : prices){
def dist = distMap.get(ps.match_region+"##"+ps.match_name)
if(dist){
def ret = []
def areaData = distCodeMap.get(dist.region)
if(areaData){
ret << ps._id.toString()
ret << dist._id.toString()
ret << dist.name
ret << cityCodeDatas.get(dist.city)
ret << dist.city
ret << areaData.district_code
ret << areaData.district
ret << ps.avg_price
ret << ps.source_type
ret << ""
ret << DateUtils.formatDateByPattern(ps.c_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(ps.c_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
rets << ret
}
}else{
println ps.match_region+"##"+ps.match_name
}
}
}
return rets;
}
}
@@ -53,25 +53,25 @@ class GuiDingPriceBatch extends BaseBatch{
}
//添加搜房的参考价
// def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]);
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]);
// def datas = findFangDatas(districts)
// def fangRets = createDatas(cityCodeDatas, distCodeMap, datas)
// rows.addAll(fangRets)
//添加保亭文昌的指导价
// def cityDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\文昌-保亭待调查小区价格 - 210902(1).xlsx")
// def cityDataMap = [:]
// for(def d : cityDatas){
// cityDataMap.put(d['A'], d['E'])
// }
// def personDatas = findPersonDatas(districts, cityDataMap);
// def personRets = createDatas(cityCodeDatas, distCodeMap, personDatas)
// rows.addAll(personRets)
def cityDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\文昌-保亭待调查小区价格 - 210902(1).xlsx")
def cityDataMap = [:]
for(def d : cityDatas){
cityDataMap.put(d['A'], d['E'])
}
def personDatas = findPersonDatas(districts, cityDataMap);
def personRets = createDatas(cityCodeDatas, distCodeMap, personDatas)
rows.addAll(personRets)
}
def headers = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\guiding_price.dat", headers, "utf-8", true);
def headers = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\guiding_price.dat", headers, "utf-8", false);
// def rets = []
// rets << headers
@@ -146,8 +146,8 @@ class GuiDingPriceBatch extends BaseBatch{
ret << ps._id.toString()
ret << dist._id.toString()
ret << dist.name
ret << cityCodeDatas.city_code
ret << cityCodeDatas.fangjia_city
ret << cityCodeDatas.get(dist.city)
ret << dist.city
ret << areaData.district_code
ret << areaData.district
ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss")
@@ -0,0 +1,98 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
class LoadGuide {
public static final DBCollection DISTRICT_GUIDE_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "district_guide_price", "mongo.db.fangjia.com", 28017)
static main(args) {
def task = new LoadGuide()
task.run()
}
def run(){
// saveDongGuangPrice("D:\\数据\\建行\\房产估值\\指导价\\东莞二手房政府指导价.xls")
// saveChengduoPrice("D:\\数据\\建行\\房产估值\\指导价\\成都部分住宅小区二手房成交参考价格201-286-258.xlsx")
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\绍兴指导价小区_0909.xlsx")
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\宁波交易参考价格.xlsx")
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\西安成交参考价格.xlsx")
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\无锡100 广州96 二手住房成交参考价格表.xlsx")
savePrice("D:\\数据\\建行\\房产估值\\指导价\\深圳指导价 - 210909 -.xlsx")
}
def savePrice(filePath){
def datas = ExcelUtils.read(filePath,0, 1)
def rets = []
for(def d : datas){
if(d['E']){
def o = new BasicDBObject()
o.city = d['E']
o.region = d['A']
o.name = d['C']
o.avg_price = d['D'].toString().toDouble().toInteger()
o.match_city = d['E']
o.match_region = d['F']
o.match_name = d['G']
o.source_type = 1 //1:成交参考价 2:参考价
o.c_date = new Date()
rets << o
}
}
if(rets){
println rets.size()
DISTRICT_GUIDE_PRICE_COLL.insert(rets)
}
}
def saveDongGuangPrice(def filePath){
def datas = ExcelUtils.read(filePath,0, 0)
def rets = []
for(def d : datas){
if(d['E']){
def o = new BasicDBObject()
o.city = d['E']
o.region = d['A']
o.name = d['B']
o.avg_price = d['D'].toString().toDouble().toInteger()
o.match_city = d['E']
o.match_region = d['F']
o.match_name = d['G']
o.house_count = d['C'].toString().toDouble().toInteger()
o.source_type = 1 //1:成交参考价 2:参考价
o.c_date = new Date()
rets << o
}
}
if(rets){
DISTRICT_GUIDE_PRICE_COLL.insert(rets)
}
}
def saveChengduoPrice(def filePath){
def datas = ExcelUtils.read(filePath,2, 0)
def rets = []
for(def d : datas){
if(d['E']){
def o = new BasicDBObject()
o.city = d['E']
o.region = d['B']
o.name = d['C']
o.avg_price = d['D'].toString().toDouble().toInteger()
o.match_city = d['E']
o.match_region = d['F']
o.match_name = d['G']
o.source_type = 1 //1:成交参考价 2:参考价
o.c_date = new Date()
rets << o
}
}
if(rets){
DISTRICT_GUIDE_PRICE_COLL.insert(rets)
}
}
}
@@ -59,7 +59,7 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw
def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0807 - 副本.xlsx"
def fileName = "d:\\数据\\建行\\房产估值\\基础数据交付情况_0903.xlsx"
//def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\小区字段.xlsx",0,1)
//def dealMap = [:]
@@ -83,58 +83,58 @@ def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
// if (city) {
// def cityid = cityCodeDatas.get(city)
// def total = findCount(cityid, sqlInstance)
// d['K'] = total
// d['M'] = total
// }
//},0)
//小区字段覆盖率
ExcelUtils.multiThreadingModify(fileName,{d->
def city = d['A']
if(city){
def cityid = cityCodeDatas.get(city)
def total = findCount(cityid, sqlInstance)
d['AA'] = total
def a = findEstateType(cityid, sqlInstance)
d['AB'] = DataHandleUtils.getDoubleRound(2, a*100/total)
def b = findFiledCount(cityid, 'buildyear', sqlInstance)
d['AC'] = DataHandleUtils.getDoubleRound(2, b*100/total)
def c = findFiledCount(cityid, 'address', sqlInstance)
d['AD'] = DataHandleUtils.getDoubleRound(2, c*100/total)
def d1 = findFiledCount(cityid, 'ZTS', sqlInstance)
d['AE'] = DataHandleUtils.getDoubleRound(2, d1*100/total)
def e = findFiledCount(cityid, 'landarea', sqlInstance)
d['AF'] = DataHandleUtils.getDoubleRound(2, e*100/total)
def f = findFiledCount(cityid, 'buildarea', sqlInstance)
d['AG'] = DataHandleUtils.getDoubleRound(2, f*100/total)
def g = findFiledCount(cityid, 'plotratio', sqlInstance)
d['AH'] = DataHandleUtils.getDoubleRound(2, g*100/total)
def h = findFiledCount(cityid, 'greeningrate', sqlInstance)
d['AI'] = DataHandleUtils.getDoubleRound(2, h*100/total)
def i = findFiledCount(cityid, 'latitude', sqlInstance)
d['AJ'] = DataHandleUtils.getDoubleRound(2, i*100/total)
def j = findFiledCount(cityid, 'parkingcnt', sqlInstance)
d['AK'] = DataHandleUtils.getDoubleRound(2, j*100/total)
println d
}
},1)
//def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0807-1.xlsx"
//def dealMap = findDealCount(sqlInstance)
////小区字段覆盖率
//ExcelUtils.multiThreadingModify(fileName,{d->
// if(d['C']){
// d['Q'] = dealMap.get(d['C'])
// def city = d['A']
// if(city){
// def cityid = cityCodeDatas.get(city)
// def total = findCount(cityid, sqlInstance)
// d['AA'] = total
//
// def a = findEstateType(cityid, sqlInstance)
// d['AB'] = DataHandleUtils.getDoubleRound(2, a*100/total)
//
// def b = findFiledCount(cityid, 'buildyear', sqlInstance)
// d['AC'] = DataHandleUtils.getDoubleRound(2, b*100/total)
//
// def c = findFiledCount(cityid, 'address', sqlInstance)
// d['AD'] = DataHandleUtils.getDoubleRound(2, c*100/total)
//
// def d1 = findFiledCount(cityid, 'ZTS', sqlInstance)
// d['AE'] = DataHandleUtils.getDoubleRound(2, d1*100/total)
//
// def e = findFiledCount(cityid, 'landarea', sqlInstance)
// d['AF'] = DataHandleUtils.getDoubleRound(2, e*100/total)
//
// def f = findFiledCount(cityid, 'buildarea', sqlInstance)
// d['AG'] = DataHandleUtils.getDoubleRound(2, f*100/total)
//
// def g = findFiledCount(cityid, 'plotratio', sqlInstance)
// d['AH'] = DataHandleUtils.getDoubleRound(2, g*100/total)
//
// def h = findFiledCount(cityid, 'greeningrate', sqlInstance)
// d['AI'] = DataHandleUtils.getDoubleRound(2, h*100/total)
//
// def i = findFiledCount(cityid, 'latitude', sqlInstance)
// d['AJ'] = DataHandleUtils.getDoubleRound(2, i*100/total)
//
// def j = findFiledCount(cityid, 'parkingcnt', sqlInstance)
// d['AK'] = DataHandleUtils.getDoubleRound(2, j*100/total)
// println d
// }
//},2)
//},1)
//def fileName = "D:\\数据\\建行\\房产估值\\数据文件\\成交统计.xlsx"
def dealMap = findDealCount(sqlInstance)
ExcelUtils.multiThreadingModify(fileName,{d->
if(d['C']){
d['K'] = dealMap.get(d['C'])
}
},2)
//def fileName = "d:\\数据\\建行\\房产估值\\经纬度错误数据修正.xlsx"
@@ -170,7 +170,7 @@ ExcelUtils.multiThreadingModify(fileName,{d->
def findDealCount(sqlInstance){
def sql = "select cityname,count(*) as count from bigdata_deal where dealdate>=to_date('2021-06-01','yyyy-mm-dd') and dealdate<to_date('2021-07-01','yyyy-mm-dd') group by cityname";
def sql = "select cityname,count(*) as count from bigdata_deal_gj where dealdate>=to_date('2021-06-01','yyyy-mm-dd') and dealdate<to_date('2021-07-01','yyyy-mm-dd') group by cityname";
def count = 0
def map = [:]
@@ -184,7 +184,7 @@ def findDealCount(sqlInstance){
def findEstateType(cityid, sqlInstance){
def sql = "select count(*) as count from bigdata_commbuildtype where cityid='${cityid}' and buildtype!='0199'";
def sql = "select count(*) as count from bigdata_commbuildtype_gj where cityid='${cityid}' and buildtype!='0199'";
def count = 0
sqlInstance.eachRow(sql,{r->
@@ -194,7 +194,7 @@ def findEstateType(cityid, sqlInstance){
}
def findCount(cityid, sqlInstance){
def sql = "select count(*) as count from bigdata_community where cityid='${cityid}'";
def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}'";
def count = 0
sqlInstance.eachRow(sql,{r->
@@ -204,7 +204,7 @@ def findCount(cityid, sqlInstance){
}
def findFiledCount(cityid, field, sqlInstance){
def sql = "select count(*) as count from bigdata_community where cityid='${cityid}' and "+field+" is not null";
def sql = "select count(*) as count from bigdata_community_gj where cityid='${cityid}' and "+field+" is not null";
def count = 0
sqlInstance.eachRow(sql,{r->
count = r.count
@@ -11,6 +11,8 @@ import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.ConcurrentHashMap
/**
* 建行模型估值项目
* 第三方均价数据同步
@@ -18,11 +20,13 @@ import com.mongodb.DBCollection
@BatchOwner(Watcher.GaoJuan)
class ThirdPartyPriceBatch extends BaseBatch {
def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def sDate= 202107//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def startDate = 202105
def endDate = 202107
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
static main(args) {
runBatch(new ThirdPartyPriceBatch(), args);
@@ -33,13 +37,15 @@ class ThirdPartyPriceBatch extends BaseBatch {
def citys = CcbCommonUtils.citys;
def rows = []
def index = 0
def weekend = DateUtils.getFirstDayOfWeek(new Date())
for(def city : citys){
index++
def statMap = getDistrictStatByCity(city, weekend)
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"])
def thirdPriceMap = findThirdPartyPrice(city)
def thirdPriceMap = findThirdPartyPrice(city, statMap)
def rets = createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap)
rows.addAll(rets)
println index+" "+city+" "+rets.size()
@@ -47,13 +53,13 @@ class ThirdPartyPriceBatch extends BaseBatch {
def fileName = "/home/goojia/ccb/third_party_price.txt"
def fileName = "/home/goojia/ccb/third_party_price.dat"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\third_party_price.txt"
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\third_party_price.dat"
}
def header = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(rows, fileName, header, "utf-8", false);
def header = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(rows, fileName, header, "utf-8", true);
}
@@ -166,10 +172,10 @@ class ThirdPartyPriceBatch extends BaseBatch {
}
def findThirdPartyPrice(def city){
def findThirdPartyPrice(def city, def statMap){
def q = new BasicDBObject()
q.put("city", city)
q.put("s_date", new BasicDBObject('$gte', sDate))
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
def map = [:]
for(def db : dbcursor){
@@ -242,13 +248,46 @@ class ThirdPartyPriceBatch extends BaseBatch {
def resultMap = [:]
map.each {key, value->
if(value && value.size()>1){
resultMap.put(key, value)
if(value.size()==2 && !value.lianjia_esf_price && !value.beike_esf_price){
def price = statMap.get(key)
if(price){
def priceMap = [:]
def startPrice = (int)(price - price*0.3)
def endPrice = (int)(price*0.3 + price)
value.each{pk,pv->
if(pk.contains("price")){
if(pv>=startPrice && pv<=endPrice){
priceMap.put(pk,pv)
}
}else{
priceMap.put(pk,pv)
}
}
if(priceMap.size()>1){
resultMap.put(key, priceMap)
}
}
}else{
resultMap.put(key, value)
}
}
}
return resultMap
}
def getDistrictStatByCity(city, weekend){
def q = new BasicDBObject()
q.put("city", city)
q.put("weekend", weekend)
def dbcursor = DIST_SELL_STAT.find(q)
def map = new ConcurrentHashMap()
for(def d : dbcursor){
map.put(d.region+"##"+d.district_name, d.avg_price)
}
return map
}
}
@@ -52,7 +52,7 @@ class YouDaDataParser {
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
// task.esfParser();//友达二手房存入mongo
task.esfParser();//友达二手房存入mongo
// task.newhouseToSaveEs();//新房保存到es
task.esfhouseToSaveEs();//二手房保存到es
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
@@ -299,14 +299,14 @@ class YouDaDataParser {
for(def i=0;i<datas.size();i++){
index++;
def data = datas[i];
fixedThreadPool.execute(
new Runnable() {
void run() {
count_.incrementAndGet()
// fixedThreadPool.execute(
// new Runnable() {
// void run() {
// count_.incrementAndGet()
saveEsfHouseToDB(data);
}
}
);
// }
// }
// );
}
latch.await()
}
@@ -14,12 +14,45 @@ class FieldStat {
static main(args) {
def task = new FieldStat();
// task.run()
task.run1()
// task.run1()
// task.run2()
task.run3()
}
def run3(){
def citys = CommonDatautils.findAllCity()//["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
def dist_total = 0
def addr_alias_total = 0
def rets = []
rets << ["城市","有地址别名总量","小区总量","有地址别名小区占比"]
for(def city : citys){
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "addr_alias"]);
def alias_index = 0
def alias_total = 0
for(def d : districts){
def alias = d.addr_alias
if(alias){
alias_index++
}
}
def precent = 0
if(districts.size()){
precent = DataHandleUtils.getDoubleRound(2, alias_index*100.0/districts.size())
}
rets << [city, alias_index, districts.size(), precent+"%"]
}
ExcelUtils.write("d:\\数据\\全国城市地址别名统计.xlsx", rets, "统计")
// println dist_total
// println alias_total
// println addr_alias_total
}
def run1(){
def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
def citys = CommonDatautils.findAllCity()//["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
def dist_total = 0
def addr_alias_total = 0
@@ -31,18 +31,24 @@ class GuangfaBlankSelectDistrict {
static main(args) {
def task = new GuangfaBlankSelectDistrict();
task.run()
// task.run()
task.run2()
}
def run(){
def file = "D:\\数据\\广发\\获取楼盘信息需求(发送版).xlsx"
def file = "D:\\数据\\广发\\获取楼盘信息需求(东莞和杭州).xlsx"
def cityMap = standardRegion(file)
def priceMap = getZhiDaoPrice(file)
selectDistricts(cityMap, priceMap)
println "写入完成"
}
def run2(){
selectDistricts2()
println "写入完成"
}
def selectDistricts(cityMap, priceMap){
def rets = []
rets << ["城市","区域","指导价","楼盘名称","楼盘地址","开发商名称","容积率","物业名称","*套数","*楼龄","户型","*建筑面积区间","2021年1-6月成交量","*均价","*成交量","*均价","*成交量","*均价"]
@@ -107,6 +113,70 @@ class GuangfaBlankSelectDistrict {
ExcelUtils.write("d:\\数据\\广发\\楼盘(西安东莞杭州).xlsx", rets, "数据")
}
def selectDistricts2(){
def rets = []
rets << ["城市","区域","指导价","楼盘名称","楼盘地址","开发商名称","容积率","物业名称","*套数","*楼龄","户型","*建筑面积区间","2021年1-6月成交量","*均价","*成交量","*均价","*成交量","*均价"]
def citys = ["东莞","杭州"]
citys.each{city->
def thirdPriceMap = findNearThirdPartyPrice(city)
def retMaps = findCreditHightDistrict3(city, "2021-08-22", thirdPriceMap)
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","address","alias","estate_company", "developer","cubage_percent","household_count", "finish_building_date"])
//基本情况合格的小区
def distMap = [:]
for(def d : districts){
// && d.household_count && d.household_count>=50
def distKeys = retMaps.get(d.region)
if(distKeys && distKeys.contains(d.city+"##"+d.region+"##"+d.name)){
if(d.finish_building_date){
def year = CcbCodeUtils.parseBuildYear(d.finish_building_date)
if(year){
def mod = 2021-year
if(mod <=15){
d.house_year = mod
}else{
d.house_year = null
}
}else{
d.house_year = null
}
}else{
d.house_year = null
}
distMap.put(d.city+"##"+d.region+"##"+d.name, d)
}
// if(d.finish_building_date){
// def year = CcbCodeUtils.parseBuildYear(d.finish_building_date)
// if(year){
// def mod = 2021-year
// if(mod <=15){
// def distKeys = retMaps.get(d.region)
// if(distKeys && distKeys.contains(d.city+"##"+d.region+"##"+d.name)){
// d.house_year = mod
// distMap.put(d.city+"##"+d.region+"##"+d.name, d)
// }
// }
// }
// }
}
println city +" 基本情况合格的小区:"+distMap.size()
for(def r : regions){
def names = retMaps.get(r)
if(names){
def results = getRoomTypeAndTradeInfo(names, distMap, priceMap)
rets.addAll(results)
}
}
println "结束"
}
println "开始写入Excel....."+rets.size()
ExcelUtils.write("d:\\数据\\广发\\楼盘(西安东莞杭州).xlsx", rets, "数据")
}
def getRoomTypeAndTradeInfo(def names, def distMap, def priceMap){
def rets = Collections.synchronizedList(new ArrayList());
@@ -332,4 +402,52 @@ class GuangfaBlankSelectDistrict {
}
return thirdPriceMap
}
def findNearThirdPartyPrice(def city){
def q = new BasicDBObject()
q.put("city", city)
q.put("s_date", new BasicDBObject('$gte', 202105))
q.put("lianjia_esf_price", new BasicDBObject(QueryOperators.EXISTS, true))
def dbcursor = AVERAGE_STATISTICS.find(q)
def thirdPriceMap = [:]
for(def d : dbcursor){
def key = d.city+"#"+d.region+"#"+d.name
if(!thirdPriceMap.containsKey(key)){
thirdPriceMap.put(d.city+"#"+d.region+"#"+d.name, d.lianjia_esf_price)
}
}
return thirdPriceMap
}
def findCreditHightDistrict3(def city, def regions, def weekend, def thirdPriceMap){
def q = new BasicDBObject()
q.put("city", city)
q.put("value_type", 2)
q.put("weekend", DateUtils.parseFromTightByPattern(weekend, "yyyy-MM-dd"))
def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("credit",-1))
def retMaps = [:]
for(def d : dbcursor){
def thirdPrice = thirdPriceMap.get(d.city +"#"+d.region+"#"+d.district_name)
if(thirdPrice){
def precent = (thirdPrice-d.avg_price)*100/d.avg_price
if(Math.abs(precent)>=0 && Math.abs(precent)<=10){
def rets = retMaps.get(d.region)
if(!rets){
rets = []
retMaps.put(d.region, rets)
}
rets << d.city+"##"+d.region+"##"+d.district_name
}
}else{
def rets = retMaps.get(d.region)
if(!rets){
rets = []
retMaps.put(d.region, rets)
}
rets << d.city+"##"+d.region+"##"+d.district_name
}
}
return retMaps;
}
}
@@ -15,9 +15,9 @@ class SynchDistrictPrice {
def static final SHORT_DATE = Utils.getLastMonth()
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
// def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房",
// "anjuke.com":"安居客","cityhouse.cn":"城市房产"]
def sourceMap = ["lianjia.com":"链家"]
def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房",
"anjuke.com":"安居客","cityhouse.cn":"城市房产"]
// def sourceMap = ["cityhouse.cn":"城市房产"]
static main(args) {
def task = new SynchDistrictPrice()
task.run()
@@ -28,23 +28,22 @@ class SynchDistrictPrice {
def citys = CommonUtils.findAllCity()
sourceMap.each { k, v ->
for(def city : citys){
if(!["上海","北京","海口"].contains(city)){
def rets = findCrawlDistrictDetails(city, k, v)
println city +" "+rets.size()
synchData(city, rets, v)
}
def rets = findCrawlDistrictDetails(city, k, v)
println city +" "+rets.size()
synchData(city, rets, v)
}
}
}
def findCrawlDistrictDetails(city, root, source){
def curMonthFirstDay = DateUtils.getFirstDayOfMonth(new Date())
def q = new BasicDBObject()
q.put("city", city)
q.put("root", root)
q.put('m_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2021-07-01")));
q.put('m_date', new BasicDBObject('$gte', curMonthFirstDay));
def keys = new BasicDBObject()
keys.put("city", 1)
keys.put("region", 1)
@@ -53,6 +52,7 @@ class SynchDistrictPrice {
keys.put("address", 1)
keys.put("avg_price", 1)
keys.put("root", 1)
keys.put("c_date", 1)
keys.put("url", 1)
def rets = []
def dbcursor = districtDetailColl.find(q, keys)
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
def run(){
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房"]
for(def source :sources){
def count1 = findTotalCount(source, 202106);
def count2 = findTotalCount(source, 202107);
def count1 = findTotalCount(source, 202107);
def count2 = findTotalCount(source, 202108);
println source +" "+count1 +" "+count2
}
}