+52
-52
@@ -37,7 +37,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"]
|
||||
|
||||
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
|
||||
@@ -61,58 +61,58 @@ class BigdataDealBatch extends BaseBatch{
|
||||
public void process(Object args) {
|
||||
//def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
|
||||
// def citys = CcbCommonUtils.citys;
|
||||
// def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
if(args && args.size()>=2){
|
||||
curDate = args[1].toString().toInteger()
|
||||
}
|
||||
println "curDate:"+curDate
|
||||
def allCitys = ["福州"] //CommonDatautils.findAllCity()
|
||||
for(def city : allCitys){
|
||||
// if(!ccbCitys.contains(city)){
|
||||
addDealPriceMatch(city);
|
||||
// }
|
||||
// def allCitys = CommonDatautils.findAllCity()
|
||||
// for(def city : allCitys){
|
||||
//// if(!ccbCitys.contains(city)){
|
||||
// addDealPriceMatch(city);
|
||||
//// }
|
||||
// }
|
||||
|
||||
|
||||
def rows = []
|
||||
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def index = 0
|
||||
for(def city : citys){
|
||||
index ++
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def ppeDists = [:]
|
||||
|
||||
def statMap = [:]//getDistrictStatByCity(city, weekend)
|
||||
if(city=="上海"){
|
||||
def youDataTrade = getYouDaData(statMap)
|
||||
def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
|
||||
rows.addAll(youdaDatas)
|
||||
println city +" "+youdaDatas.size()
|
||||
}
|
||||
|
||||
def trades = getCityData(city, statMap, ppeDists)
|
||||
if(trades){
|
||||
def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
|
||||
rows.addAll(rets)
|
||||
println "【${index}】"+city +" 成交共:"+rets.size()
|
||||
|
||||
def cityDatas = []
|
||||
cityDatas << headers
|
||||
cityDatas.addAll(rows)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
|
||||
}
|
||||
|
||||
// def rows = []
|
||||
// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
// def index = 0
|
||||
// for(def city : citys){
|
||||
// index ++
|
||||
// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
// def ppeDists = [:]
|
||||
//
|
||||
// def statMap = [:]//getDistrictStatByCity(city, weekend)
|
||||
// if(city=="上海"){
|
||||
// def youDataTrade = getYouDaData(statMap)
|
||||
// def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
|
||||
// rows.addAll(youdaDatas)
|
||||
// println city +" "+youdaDatas.size()
|
||||
// }
|
||||
//
|
||||
// def trades = getCityData(city, statMap, ppeDists)
|
||||
// if(trades){
|
||||
// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
|
||||
// rows.addAll(rets)
|
||||
// println "【${index}】"+city +" 成交共:"+rets.size()
|
||||
//
|
||||
// def cityDatas = []
|
||||
// cityDatas << headers
|
||||
// cityDatas.addAll(rows)
|
||||
// }
|
||||
//
|
||||
// }
|
||||
//
|
||||
// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
|
||||
// if(CommonDatautils.isWindow()){
|
||||
// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
|
||||
// }
|
||||
//
|
||||
// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
|
||||
// println "txt 生成 " + rows.size()
|
||||
//
|
||||
// saveToPgSql(rows)
|
||||
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
|
||||
println "txt 生成 " + rows.size()
|
||||
|
||||
saveToPgSql(rows)
|
||||
}
|
||||
|
||||
def createData(def rets, cityCodeDatas, distCodeMap, source){
|
||||
@@ -652,9 +652,9 @@ class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
for(def i=0;i<trades.size();i++){
|
||||
final def t = trades[i]
|
||||
// fixedThreadPool.execute(
|
||||
// new Runnable() {
|
||||
// void run() {
|
||||
fixedThreadPool.execute(
|
||||
new Runnable() {
|
||||
void run() {
|
||||
count_.addAndGet(1)
|
||||
try{
|
||||
if(!t.match_id){
|
||||
@@ -690,9 +690,9 @@ class BigdataDealBatch extends BaseBatch{
|
||||
latch.countDown()
|
||||
}
|
||||
|
||||
// }
|
||||
// }
|
||||
// );
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
latch.await()
|
||||
println city +" "+trades.size()+" "+new Date().toString()
|
||||
|
||||
+3
-3
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2023-05-27", "2023-06-26")
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2023-06-26", "2023-07-20")
|
||||
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
@@ -84,8 +84,8 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
|
||||
saveToPgSql(rows)
|
||||
//
|
||||
// //更新静态数据表
|
||||
// updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28")
|
||||
//更新静态数据表
|
||||
updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28")
|
||||
|
||||
}
|
||||
|
||||
|
||||
+5
-5
@@ -93,13 +93,13 @@ class TradeStatLog {
|
||||
def run(){
|
||||
def citys = CcbCommonUtils.citys_all;
|
||||
def rets = []
|
||||
rets << ["城市","23年3月总量","23年4月总量","23年5月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
rets << ["城市","23年4月总量","23年5月总量","23年6月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"]
|
||||
//def startDate = "2021-05-01"
|
||||
def startDate = "2023-03-01"
|
||||
def date = "2023-04-01"
|
||||
def endDate = "2023-05-01"
|
||||
def newDate = "2023-06-01"
|
||||
def startDate = "2023-04-01"
|
||||
def date = "2023-05-01"
|
||||
def endDate = "2023-06-01"
|
||||
def newDate = "2023-07-01"
|
||||
|
||||
for(def city : citys){
|
||||
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
|
||||
|
||||
@@ -295,6 +295,23 @@ class FiledStrand {
|
||||
return null
|
||||
}
|
||||
|
||||
static def processEstateType(estateType){
|
||||
if(estateType){
|
||||
if(estateType.contains("纯写字楼") || estateType.contains("商务中心")){
|
||||
return "标准写字楼"
|
||||
}else if(estateType.contains("SOHO") || estateType.contains("商住") || estateType.contains("商用") || estateType.contains("LOFT") || estateType.contains("酒店")){
|
||||
return "商住两用"
|
||||
}else if(estateType.contains("园区")){
|
||||
return "创意园区"
|
||||
}else if(estateType.contains("综合体")){
|
||||
return "综合体"
|
||||
}else{
|
||||
return "其他"
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static def processStandardArea(area){
|
||||
if(area){
|
||||
area = area.toString().replace(",",",").replace("㎡", "").replace("m", "").replace("/","")
|
||||
@@ -684,7 +701,7 @@ class FiledStrand {
|
||||
}else{
|
||||
return null;
|
||||
}
|
||||
}else if(estateType2=="写字楼"){
|
||||
}else if(estateType2=="写字楼" || estateType2=="office"){
|
||||
if(estateCharege>=0.35 && estateCharege<=60){
|
||||
return estateCharege;
|
||||
}else{
|
||||
|
||||
+22
-8
@@ -36,10 +36,10 @@ class OfficeFieldCleanBatch extends BaseBatch{
|
||||
"virescence_percent":"greening_rate","cubage_percent":"plot_ratio","build_area":"build_area","all_area":"land_area"
|
||||
,"finish_building_date":"finish_building_date","property_life":"property_life","standard_area":"standard_area","total_floor":"total_floor",
|
||||
"standard_height":"floor_height", "real_use_percent":"room_rate", "loop_line_info":"ring_location","car_site_count":"parking",
|
||||
"parking_fee":"car_fee","elevators_count":"elevator_num","bay_area":"open_area"]//key:seaweed表字段 value:es表字段
|
||||
"parking_fee":"car_fee","elevators_count":"elevator_num","bay_area":"open_area","estate_type2":"property_type"]//key:seaweed表字段 value:es表字段
|
||||
|
||||
//"安居客":"安居客app",
|
||||
def sourceMap = ["房天下":"房天下","楼盘网":"楼盘网","办办网":"办办网","点点租":"点点租"]
|
||||
//
|
||||
def sourceMap = ["安居客":"安居客app","房天下":"房天下","楼盘网":"楼盘网","办办网":"办办网","点点租":"点点租"]
|
||||
|
||||
def fieldMap = ["编号":"_id","城市":"city","区域":"region","名称":"name","别名":"alias","租金":"rent_price","板块":"block","物业公司":"estate_company","物业费":"estate_charge",
|
||||
"地址":"address","地址别名":"addr_alias","经度":"lng","纬度":"lat","类型":"estate_type2","建筑面积":"build_area","等级":"bed_kind",
|
||||
@@ -71,11 +71,11 @@ class OfficeFieldCleanBatch extends BaseBatch{
|
||||
|
||||
// addBlock()
|
||||
|
||||
// addFieldValue()
|
||||
addFieldValue()
|
||||
|
||||
// updateErrorField()
|
||||
export()
|
||||
test()
|
||||
// export()
|
||||
// test()
|
||||
// addOffice()
|
||||
}
|
||||
|
||||
@@ -478,7 +478,12 @@ class OfficeFieldCleanBatch extends BaseBatch{
|
||||
if(esVal){
|
||||
def newVal = null
|
||||
if(seaweedfield == "estate_charge"){
|
||||
newVal = FiledStrand.processEstateCharge(c, FiledStrand.getNumber(esVal), dist.estate_type2)
|
||||
if(dist.cat == "office"){
|
||||
newVal = FiledStrand.processEstateCharge(c, FiledStrand.getNumber(esVal), "office")
|
||||
}else{
|
||||
newVal = FiledStrand.processEstateCharge(c, FiledStrand.getNumber(esVal), dist.estate_type2)
|
||||
}
|
||||
|
||||
newVal = newVal.toString()
|
||||
}
|
||||
if(seaweedfield == "build_date" || seaweedfield == "live_date"){
|
||||
@@ -532,7 +537,12 @@ class OfficeFieldCleanBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
if(seaweedfield == "build_type"){
|
||||
newVal = FiledStrand.processBuildType(esVal, dist.estate_type2)
|
||||
if(dist.cat == "office") {
|
||||
newVal = FiledStrand.processBuildType(esVal, "office")
|
||||
}else{
|
||||
newVal = FiledStrand.processBuildType(esVal, dist.estate_type2)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(seaweedfield == "land_property"){
|
||||
@@ -570,6 +580,10 @@ class OfficeFieldCleanBatch extends BaseBatch{
|
||||
newVal = esVal
|
||||
}
|
||||
|
||||
if(seaweedfield == "estate_type2"){
|
||||
newVal = FiledStrand.processEstateType(esVal)
|
||||
}
|
||||
|
||||
if(newVal){
|
||||
newValues.put(seaweedfield, newVal)
|
||||
}
|
||||
|
||||
+4
-1
@@ -296,7 +296,10 @@ public class BeiKeCrawlByChrome {
|
||||
def detailDiv1 = JQueryUtils.select(".introContent .base", detailDoc)
|
||||
def detailDiv2 = JQueryUtils.select(".introContent .transaction", detailDoc)
|
||||
|
||||
def liftStr = detailDiv1.select("li:contains(梯户比例)").text().replace("梯户比例","")
|
||||
def liftStr = null;
|
||||
if(detailDiv1){
|
||||
liftStr = detailDiv1.select("li:contains(梯户比例)").text().replace("梯户比例","")
|
||||
}
|
||||
if(liftStr){
|
||||
detail.lift = DataFormatUtils.parseLift(liftStr)
|
||||
detail.lift_household = DataFormatUtils.parseHousehold(liftStr)
|
||||
|
||||
Reference in New Issue
Block a user