This commit is contained in:
2023-07-20 16:09:17 +08:00
parent 41d099fcb5
commit 7a52c6764c
6 changed files with 104 additions and 70 deletions
@@ -37,7 +37,7 @@ class BigdataDealBatch extends BaseBatch{
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"]
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
@@ -61,58 +61,58 @@ class BigdataDealBatch extends BaseBatch{
public void process(Object args) {
//def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
// def citys = CcbCommonUtils.citys;
// def citys = CcbCommonUtils.getRunningCitys(args)
def citys = CcbCommonUtils.getRunningCitys(args)
if(args && args.size()>=2){
curDate = args[1].toString().toInteger()
}
println "curDate:"+curDate
def allCitys = ["福州"] //CommonDatautils.findAllCity()
for(def city : allCitys){
// if(!ccbCitys.contains(city)){
addDealPriceMatch(city);
// }
// def allCitys = CommonDatautils.findAllCity()
// for(def city : allCitys){
//// if(!ccbCitys.contains(city)){
// addDealPriceMatch(city);
//// }
// }
def rows = []
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def index = 0
for(def city : citys){
index ++
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def ppeDists = [:]
def statMap = [:]//getDistrictStatByCity(city, weekend)
if(city=="上海"){
def youDataTrade = getYouDaData(statMap)
def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
rows.addAll(youdaDatas)
println city +" "+youdaDatas.size()
}
def trades = getCityData(city, statMap, ppeDists)
if(trades){
def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
rows.addAll(rets)
println "【${index}】"+city +" 成交共:"+rets.size()
def cityDatas = []
cityDatas << headers
cityDatas.addAll(rows)
}
}
def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
}
// def rows = []
// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
// def index = 0
// for(def city : citys){
// index ++
// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
// def ppeDists = [:]
//
// def statMap = [:]//getDistrictStatByCity(city, weekend)
// if(city=="上海"){
// def youDataTrade = getYouDaData(statMap)
// def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
// rows.addAll(youdaDatas)
// println city +" "+youdaDatas.size()
// }
//
// def trades = getCityData(city, statMap, ppeDists)
// if(trades){
// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
// rows.addAll(rets)
// println "【${index}】"+city +" 成交共:"+rets.size()
//
// def cityDatas = []
// cityDatas << headers
// cityDatas.addAll(rows)
// }
//
// }
//
// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
// if(CommonDatautils.isWindow()){
// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
// }
//
// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
// println "txt 生成 " + rows.size()
//
// saveToPgSql(rows)
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
println "txt 生成 " + rows.size()
saveToPgSql(rows)
}
def createData(def rets, cityCodeDatas, distCodeMap, source){
@@ -652,9 +652,9 @@ class BigdataDealBatch extends BaseBatch{
for(def i=0;i<trades.size();i++){
final def t = trades[i]
// fixedThreadPool.execute(
// new Runnable() {
// void run() {
fixedThreadPool.execute(
new Runnable() {
void run() {
count_.addAndGet(1)
try{
if(!t.match_id){
@@ -690,9 +690,9 @@ class BigdataDealBatch extends BaseBatch{
latch.countDown()
}
// }
// }
// );
}
}
);
}
latch.await()
println city +" "+trades.size()+" "+new Date().toString()
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2023-05-27", "2023-06-26")
def prices = CcbCommonUtils.findPersonPrice(city, "2023-06-26", "2023-07-20")
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
if(prices){
def distMap = getDistrictMap(city)
@@ -84,8 +84,8 @@ class GuiDingPriceBatch extends BaseBatch{
saveToPgSql(rows)
//
// //更新静态数据表
// updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28")
//更新静态数据表
updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28")
}
@@ -93,13 +93,13 @@ class TradeStatLog {
def run(){
def citys = CcbCommonUtils.citys_all;
def rets = []
rets << ["城市","23年3月总量","23年4月总量","23年5月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
rets << ["城市","23年4月总量","23年5月总量","23年6月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"]
//def startDate = "2021-05-01"
def startDate = "2023-03-01"
def date = "2023-04-01"
def endDate = "2023-05-01"
def newDate = "2023-06-01"
def startDate = "2023-04-01"
def date = "2023-05-01"
def endDate = "2023-06-01"
def newDate = "2023-07-01"
for(def city : citys){
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
@@ -295,6 +295,23 @@ class FiledStrand {
return null
}
static def processEstateType(estateType){
if(estateType){
if(estateType.contains("纯写字楼") || estateType.contains("商务中心")){
return "标准写字楼"
}else if(estateType.contains("SOHO") || estateType.contains("商住") || estateType.contains("商用") || estateType.contains("LOFT") || estateType.contains("酒店")){
return "商住两用"
}else if(estateType.contains("园区")){
return "创意园区"
}else if(estateType.contains("综合体")){
return "综合体"
}else{
return "其他"
}
}
return null
}
static def processStandardArea(area){
if(area){
area = area.toString().replace(",",",").replace("㎡", "").replace("m", "").replace("/","")
@@ -684,7 +701,7 @@ class FiledStrand {
}else{
return null;
}
}else if(estateType2=="写字楼"){
}else if(estateType2=="写字楼" || estateType2=="office"){
if(estateCharege>=0.35 && estateCharege<=60){
return estateCharege;
}else{
@@ -36,10 +36,10 @@ class OfficeFieldCleanBatch extends BaseBatch{
"virescence_percent":"greening_rate","cubage_percent":"plot_ratio","build_area":"build_area","all_area":"land_area"
,"finish_building_date":"finish_building_date","property_life":"property_life","standard_area":"standard_area","total_floor":"total_floor",
"standard_height":"floor_height", "real_use_percent":"room_rate", "loop_line_info":"ring_location","car_site_count":"parking",
"parking_fee":"car_fee","elevators_count":"elevator_num","bay_area":"open_area"]//key:seaweed表字段 value:es表字段
"parking_fee":"car_fee","elevators_count":"elevator_num","bay_area":"open_area","estate_type2":"property_type"]//key:seaweed表字段 value:es表字段
//"安居客":"安居客app",
def sourceMap = ["房天下":"房天下","楼盘网":"楼盘网","办办网":"办办网","点点租":"点点租"]
//
def sourceMap = ["安居客":"安居客app","房天下":"房天下","楼盘网":"楼盘网","办办网":"办办网","点点租":"点点租"]
def fieldMap = ["编号":"_id","城市":"city","区域":"region","名称":"name","别名":"alias","租金":"rent_price","板块":"block","物业公司":"estate_company","物业费":"estate_charge",
"地址":"address","地址别名":"addr_alias","经度":"lng","纬度":"lat","类型":"estate_type2","建筑面积":"build_area","等级":"bed_kind",
@@ -71,11 +71,11 @@ class OfficeFieldCleanBatch extends BaseBatch{
// addBlock()
// addFieldValue()
addFieldValue()
// updateErrorField()
export()
test()
// export()
// test()
// addOffice()
}
@@ -478,7 +478,12 @@ class OfficeFieldCleanBatch extends BaseBatch{
if(esVal){
def newVal = null
if(seaweedfield == "estate_charge"){
newVal = FiledStrand.processEstateCharge(c, FiledStrand.getNumber(esVal), dist.estate_type2)
if(dist.cat == "office"){
newVal = FiledStrand.processEstateCharge(c, FiledStrand.getNumber(esVal), "office")
}else{
newVal = FiledStrand.processEstateCharge(c, FiledStrand.getNumber(esVal), dist.estate_type2)
}
newVal = newVal.toString()
}
if(seaweedfield == "build_date" || seaweedfield == "live_date"){
@@ -532,7 +537,12 @@ class OfficeFieldCleanBatch extends BaseBatch{
}
if(seaweedfield == "build_type"){
newVal = FiledStrand.processBuildType(esVal, dist.estate_type2)
if(dist.cat == "office") {
newVal = FiledStrand.processBuildType(esVal, "office")
}else{
newVal = FiledStrand.processBuildType(esVal, dist.estate_type2)
}
}
if(seaweedfield == "land_property"){
@@ -570,6 +580,10 @@ class OfficeFieldCleanBatch extends BaseBatch{
newVal = esVal
}
if(seaweedfield == "estate_type2"){
newVal = FiledStrand.processEstateType(esVal)
}
if(newVal){
newValues.put(seaweedfield, newVal)
}
@@ -296,7 +296,10 @@ public class BeiKeCrawlByChrome {
def detailDiv1 = JQueryUtils.select(".introContent .base", detailDoc)
def detailDiv2 = JQueryUtils.select(".introContent .transaction", detailDoc)
def liftStr = detailDiv1.select("li:contains(梯户比例)").text().replace("梯户比例","")
def liftStr = null;
if(detailDiv1){
liftStr = detailDiv1.select("li:contains(梯户比例)").text().replace("梯户比例","")
}
if(liftStr){
detail.lift = DataFormatUtils.parseLift(liftStr)
detail.lift_household = DataFormatUtils.parseHousehold(liftStr)