+38
-38
@@ -52,7 +52,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
//def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
|
||||
def citys = ['南京', '无锡', '徐州', '常州', '苏州', '南通', '连云港', '淮安', '盐城', '扬州', '镇江', '泰州', '宿迁']//CcbCommonUtils.citys;
|
||||
def citys = CcbCommonUtils.citys;
|
||||
// def ccbCitys = CcbCommonUtils.citys;
|
||||
|
||||
for(def city : citys){
|
||||
@@ -60,39 +60,39 @@ class BigdataDealBatch extends BaseBatch{
|
||||
addDealPriceMatch(city);
|
||||
// }
|
||||
}
|
||||
// def rows = []
|
||||
// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
// for(def city : citys){
|
||||
// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
// def ppeDists = [:]
|
||||
//
|
||||
// def statMap = [:]//getDistrictStatByCity(city, weekend)
|
||||
// if(city=="上海"){
|
||||
// def youDataTrade = getYouDaData(statMap)
|
||||
// def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
|
||||
// rows.addAll(youdaDatas)
|
||||
// println city +" "+youdaDatas.size()
|
||||
// }
|
||||
//
|
||||
//
|
||||
// def trades = getCityData(city, statMap, ppeDists)
|
||||
// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
|
||||
// rows.addAll(rets)
|
||||
// println city +" 成交共:"+rets.size()
|
||||
//
|
||||
// def cityDatas = []
|
||||
// cityDatas << headers
|
||||
// cityDatas.addAll(rows)
|
||||
// }
|
||||
//
|
||||
// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
|
||||
// if(CommonDatautils.isWindow()){
|
||||
// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
|
||||
// }
|
||||
//
|
||||
// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
|
||||
// println "txt 生成"
|
||||
def rows = []
|
||||
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
for(def city : citys){
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def ppeDists = [:]
|
||||
|
||||
def statMap = [:]//getDistrictStatByCity(city, weekend)
|
||||
if(city=="上海"){
|
||||
def youDataTrade = getYouDaData(statMap)
|
||||
def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
|
||||
rows.addAll(youdaDatas)
|
||||
println city +" "+youdaDatas.size()
|
||||
}
|
||||
|
||||
|
||||
def trades = getCityData(city, statMap, ppeDists)
|
||||
def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
|
||||
rows.addAll(rets)
|
||||
println city +" 成交共:"+rets.size()
|
||||
|
||||
def cityDatas = []
|
||||
cityDatas << headers
|
||||
cityDatas.addAll(rows)
|
||||
}
|
||||
|
||||
def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
|
||||
}
|
||||
|
||||
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
|
||||
println "txt 生成"
|
||||
|
||||
|
||||
}
|
||||
@@ -569,10 +569,10 @@ class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
|
||||
def addDealPriceMatch(city){
|
||||
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210101", "yyyyMMdd"))
|
||||
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20220601", "yyyyMMdd"))
|
||||
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210101", "yyyyMMdd"))
|
||||
// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20220601", "yyyyMMdd"))
|
||||
def trades = CcbCommonUtils.findTradesByMonth(city, startDate, endDate)
|
||||
|
||||
final CountDownLatch latch = new CountDownLatch(trades.size())
|
||||
|
||||
+3
-3
@@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class ThirdPartyPriceBatch extends BaseBatch {
|
||||
|
||||
def sDate= 202206//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def startDate = 202204
|
||||
def endDate = 202206
|
||||
def sDate= 202207//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def startDate = 202205
|
||||
def endDate = 202207
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
+1
-1
@@ -34,7 +34,7 @@ public class LianjiaCrawlByChrome {
|
||||
|
||||
static main(args) {
|
||||
def task = new LianjiaCrawlByChrome();
|
||||
task.run()
|
||||
// task.run()
|
||||
// task.run1()
|
||||
task.addDealRegion();
|
||||
task.standardRegion()
|
||||
|
||||
+55
-1
@@ -32,8 +32,27 @@ class MingShengStat {
|
||||
def task = new MingShengStat();
|
||||
// task.run()
|
||||
// task.addNewLevel
|
||||
task.addHouseRate()
|
||||
// task.addHouseRate()
|
||||
// task.houseFieldStat()
|
||||
task.addDistBuildStr()
|
||||
// task.statFloorInfo()
|
||||
}
|
||||
|
||||
def addDistBuildStr(){
|
||||
def citys = ["北京","上海","广州","深圳","武汉","大连","南京","杭州","太原","石家庄","重庆","西安","福州","济南","宁波","成都","天津","昆明","南昌","青岛","厦门","郑州","长沙","长春","合肥","呼和浩特","沈阳","贵阳","拉萨","宝鸡","汕头","苏州","温州","泉州","邯郸","唐山","沧州","洛阳","南阳","襄阳","吕梁","大同","潍坊","南通","衡阳","中山","宜昌","许昌","盐城","济宁","龙岩","上饶","马鞍山","赣州","金华","常德","延边","泰州","三亚","淮安","咸宁","无锡","常州","镇江","烟台","绍兴","莆田","东莞","佛山","株洲","黄石","孝感","香港"]
|
||||
|
||||
def rows = []
|
||||
for(def c : citys){
|
||||
def row = []
|
||||
row << c
|
||||
def cityInfoDto = CommonDatautils.findCityInfo(c)
|
||||
if(cityInfoDto){
|
||||
def detailMap = cityInfoDto.district_detail
|
||||
row << detailMap.finishBuildingDate.toDouble()*100
|
||||
}
|
||||
rows << row
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\民生\\字段覆盖率统计-竣工时间.xlsx", rows, "字段统计")
|
||||
}
|
||||
|
||||
def houseFieldStat(){
|
||||
@@ -58,6 +77,25 @@ class MingShengStat {
|
||||
ExcelUtils.write("D:\\数据\\民生\\字段覆盖率统计2.xlsx", rows, "字段统计")
|
||||
}
|
||||
|
||||
|
||||
|
||||
def statFloorInfo(){
|
||||
def file = "D:\\数据\\民生\\字段覆盖率统计-楼层.xlsx"
|
||||
ExcelUtils.multiThreadingModify(file,{d->
|
||||
def total = findHouseCount(d['A'])
|
||||
if(total>=100000){
|
||||
total = 100000
|
||||
}
|
||||
def num = findHouseDatas(d['A']);
|
||||
if(num){
|
||||
def rate = com.fangjia.script.common.DataHandleUtils.getDoubleRound(4, num*100/total)
|
||||
d['C'] = rate
|
||||
}else{
|
||||
d['C'] = 0
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addHouseRate(){
|
||||
// def datas = ExcelUtils.read("C:\\Users\\gaojuan\\Downloads\\估值数据统计结果20220816.xlsx",0,1)
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\民生\\城市统计.xlsx",{d->
|
||||
@@ -82,6 +120,22 @@ class MingShengStat {
|
||||
return count
|
||||
}
|
||||
|
||||
|
||||
def findHouseDatas(city){
|
||||
def sql = "select floor_info from seawater where p_date >= 20200801 and city='${city}' and cat='sell' order by p_date desc limit 100000"
|
||||
def f_count = 0
|
||||
def count = 0
|
||||
ElasticUtils.executeSqlQuery(sql, {
|
||||
count++
|
||||
def s = it.floor_info
|
||||
if(s && !s.contains("暂")){
|
||||
f_count = f_count +1
|
||||
}
|
||||
println("${count} $city\t floor_info ${s}")
|
||||
})
|
||||
return f_count
|
||||
}
|
||||
|
||||
def findHouseCount(city){
|
||||
def count = 0
|
||||
ElasticUtils.executeSqlQuery("select count(*) as count from seawater where p_date >= 20200801 and city='${city}' and cat='sell'", {
|
||||
|
||||
+3
-1
@@ -24,7 +24,7 @@ class SynchDistrictPrice {
|
||||
|
||||
//,"城市房产":"城市房产"
|
||||
//"365淘房":"365淘房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","我爱我家":"我爱我家","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房",
|
||||
def esSources = ["乐有家":"乐有家","安居客小程序":"安居客"]//"吉屋":"吉屋","365淘房":"365淘房"
|
||||
def esSources = ["C21":"C21","365淘房":"365淘房"]//"吉屋":"吉屋","365淘房":"365淘房"
|
||||
static main(args) {
|
||||
def task = new SynchDistrictPrice()
|
||||
// task.run()
|
||||
@@ -33,6 +33,8 @@ class SynchDistrictPrice {
|
||||
|
||||
def run1(){
|
||||
def citys = CommonUtils.findAllCity()
|
||||
// citys.remove("上海")
|
||||
// citys.remove("广州")
|
||||
|
||||
esSources.each{esSource, mgSource->
|
||||
for(def city : citys){
|
||||
|
||||
Reference in New Issue
Block a user