From b1f117c18f07cf75ea3029387512bd80dd3f799e Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 8 Sep 2021 16:15:13 +0800 Subject: [PATCH] update --- .../BigdataCommbuildtypeBatch.groovy | 2 +- .../ccbevaluation/BigdataDealBatch.groovy | 2 + .../ccbevaluation/BigdataDistrictBatch.groovy | 2 +- .../ccbevaluation/CcbCommonUtils.groovy | 4 +- .../ccbevaluation/EvaluateRateBatch.groovy | 2 +- .../ccbevaluation/GuiDingPriceBatch.groovy | 131 ++++++++++++++++-- .../ccbevaluation/StagingDistrictBatch.groovy | 4 +- .../script/commData/CommonDatautils.groovy | 1 + 8 files changed, 132 insertions(+), 16 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy index c1126334f..e6b9defe3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy @@ -39,7 +39,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ println city +" "+rows.size() } def headers = ["城市ID","小区ID"," 建筑类型","备注","创建时间","更新时间","来源公司","数据时间"] - FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_commbuildtype.txt",headers, "utf-8", false); + FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_commbuildtype.dat",headers, "utf-8", false); println "txt 生成" } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 82766d351..0f3230368 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -49,6 +49,7 @@ class BigdataDealBatch extends BaseBatch{ def headers = ["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() for(def city : citys){ + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def statMap = getDistrictStatByCity(city, weekend) @@ -445,6 +446,7 @@ class BigdataDealBatch extends BaseBatch{ def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) def q = new BasicDBObject() q.put("contractTime", new BasicDBObject('$gte', startDate)); + q.put("area", new BasicDBObject('$gte', 20)) def dbcursor = youda_esfhouse_coll.find(q).batchSize(2000) def rets = [] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDistrictBatch.groovy index 815336ae3..08c478e6d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDistrictBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDistrictBatch.groovy @@ -41,7 +41,7 @@ class BigdataDistrictBatch extends BaseBatch{ rets << headers rets.addAll(allDatas) ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_district.xlsx", rets, "数据") - FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_district.txt",headers, "utf-8",false); + FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_district.dat",headers, "utf-8",false); } def createDatas(rows){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index e80d7fd22..9a41d91ad 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -79,7 +79,7 @@ class CcbCommonUtils { def q = new BasicDBObject() q.put("city", dist.city) q.put("region", dist.region) - + q.put("area", new BasicDBObject('$gte', 20)) def alias = [] if(dist.alias){ if(!dist.alias.contains(dist.name)){ @@ -174,7 +174,7 @@ class CcbCommonUtils { q.put("visible",0) q.put("cat", cat); q.put("city", city); -// q.put("name", "美辰香醒") +// q.put("name", "石泉六村") // q.put("name_cat", 0); // q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"])) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/EvaluateRateBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/EvaluateRateBatch.groovy index d64c08dd6..7b40980e1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/EvaluateRateBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/EvaluateRateBatch.groovy @@ -30,7 +30,7 @@ class EvaluateRateBatch extends BaseBatch{ rets << headers rets.addAll(allDatas) ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\evaluate_rate.xlsx", rets, "数据") - FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\evaluate_rate.txt",headers, "utf-8", false); + FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\evaluate_rate.dat",headers, "utf-8", false); } def createDatas(def datas){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 49c60f702..1842c3e5b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -4,13 +4,20 @@ import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.CcbCodeUtils import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors - +import java.util.concurrent.atomic.AtomicLong +import org.bson.types.ObjectId /** * 建行模型估值项目 * 人工指导价同步 @@ -20,6 +27,8 @@ import java.util.concurrent.Executors class GuiDingPriceBatch extends BaseBatch{ public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40) + def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); + def static final DBCollection dataProcessColl = CrawalDBUtil.getDataProcessCol(); static main(args) { runBatch(new GuiDingPriceBatch(), args); @@ -34,6 +43,7 @@ class GuiDingPriceBatch extends BaseBatch{ def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + //添加自己库中的人工指导价 def prices = CcbCommonUtils.findPersonPrice(city, DateUtils.addMonth(new Date(), -6)) if(prices){ def distMap = getDistrictMap(city) @@ -42,18 +52,88 @@ class GuiDingPriceBatch extends BaseBatch{ println city + " " + rets.size() } + //添加搜房的参考价 +// def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]); +// def datas = findFangDatas(districts) +// def fangRets = createDatas(cityCodeDatas, distCodeMap, datas) +// rows.addAll(fangRets) + + //添加保亭文昌的指导价 +// def cityDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\文昌-保亭待调查小区价格 - 210902(1).xlsx") +// def cityDataMap = [:] +// for(def d : cityDatas){ +// cityDataMap.put(d['A'], d['E']) +// } +// def personDatas = findPersonDatas(districts, cityDataMap); +// def personRets = createDatas(cityCodeDatas, distCodeMap, personDatas) +// rows.addAll(personRets) + } - def headers = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"] - FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\guiding_price.txt", headers, "utf-8", false); + def headers = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"] + FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\guiding_price.dat", headers, "utf-8", true); - def rets = [] - rets << headers - rets.addAll(rows) - ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\guiding_price.xlsx", rets, "数据") +// def rets = [] +// rets << headers +// rets.addAll(rows) +// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\guiding_price.xlsx", rets, "数据") } + def findFangDatas(def districts){ + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(districts.size()) + println " 小区量:"+districts.size() + def result = Collections.synchronizedList(new ArrayList()); + for(def i=0;i