From f899f50de345ef937f361fb23fe8f6ba1bc18a17 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 21 Jul 2023 18:32:06 +0800 Subject: [PATCH] =?UTF-8?q?=E7=A6=8F=E5=B7=9E=E8=84=9A=E6=9C=AC=E4=BF=AE?= =?UTF-8?q?=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/BigdataDealBatch.groovy | 2 +- .../ccbevaluation/GuiDingPriceBatch.groovy | 2 +- .../script/commData/CommonDatautils.groovy | 24 +++++++++++- .../fuzhou/DistrictInfoExportBatch.groovy | 37 ++++++++++++++---- .../fuzhou/DistrictPriceExportBatch.groovy | 24 ++++++++++-- .../fuzhou/DistrictSellStatBatch.groovy | 39 +++++++++++++------ .../fangjia/script/gj/poc/AddPriceTest.groovy | 14 +++---- 7 files changed, 109 insertions(+), 33 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 2b9cba6de..1f2867d17 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -141,7 +141,7 @@ class BigdataDealBatch extends BaseBatch{ totalCount.addAndGet(1) if(dealIds.contains(trade._id.toString())){ - println "重复:"+trade._id.toString() + " "+ trade.district_name + println "重复:"+trade._id.toString() + " "+ trade.city+ " " +trade.district_name }else{ if(source == "youda"){ def ret = converYouDaData(trade, cityCodeDatas, distCodeMap, ccbPriceMap) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index a27d6a015..ec06ada4d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2023-06-26", "2023-07-20") + def prices = CcbCommonUtils.findPersonPrice(city, "2023-06-26", "2023-07-22") // def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") if(prices){ def distMap = getDistrictMap(city) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 454b1a9e9..b889dd3ab 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -252,7 +252,6 @@ class CommonDatautils { q.put("visible", 0); q.put("cat", cat); q.put("city", city); -// q.put("name", "交发风景") DBObject keys = new BasicDBObject(); if(fields){ @@ -270,6 +269,29 @@ class CommonDatautils { } + public static def findAllUpdateDistricts(def city, def cat, def fields, def startMdate, def endMdate){ + DBObject q = new BasicDBObject(); + q.put("visible", 0); + q.put("cat", cat); + q.put("city", city); + q.put("m_date", new BasicDBObject('$gte', startMdate).append('$lt', endMdate)) + + DBObject keys = new BasicDBObject(); + if(fields){ + for(def field : fields){ + keys.put(field, 1); + } + } + + DSCursor cursor = getDataStoreConnection().find(q, keys).batchSize(2000) + List rets = [] + for(def obj : cursor){ + println obj.name +" "+obj.m_date + rets << obj + } + return rets; + } + public static def findRemoveDistricts(def city, def cat, def fields, def m_date){ DBObject q = new BasicDBObject(); q.put("status", 1); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy index 414e61113..5815f2e30 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy @@ -25,7 +25,7 @@ import java.util.regex.Pattern @BatchOwner(Watcher.GaoJuan) class DistrictInfoExportBatch extends BaseBatch{ - def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); +// def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def charset = "GBK" @@ -41,6 +41,9 @@ class DistrictInfoExportBatch extends BaseBatch{ @Override void process(Object args) { + def curDate = Integer.parseInt(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")); + + def headers = ["id","city","district","name","lng","lat","property_type","build_type","households", "property_fee","car_site_count","property_life","house_property","open_date","finish_date", "delivery_date","floor_area","build_area","plot_ratio","landscaping_ratio","developer","property_company", @@ -74,19 +77,35 @@ class DistrictInfoExportBatch extends BaseBatch{ def createDatas(def city){ def result = [] - def districts = CommonDatautils.findAllDistricts(city, "district", null); + def districts = null - def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01"); + def day = new Date().getDate() + def startMDate = DateUtils.truncateDay(new Date()-1) + def endMDate = DateUtils.truncateDay(new Date()) + if(day == 15){ + districts = CommonDatautils.findAllDistricts(city, "district", null); + }else{ + districts = CommonDatautils.findAllUpdateDistricts(city, "district", null, startMDate, endMDate); + } def lastDate = DateUtils.getLastDayOfMonth(new Date(), -1) - def startDate = DateUtils.getFirstDayOfMonth(lastDate); +// def startDate = DateUtils.getFirstDayOfMonth(lastDate); def fzAddrMap = getFzAddress(); //因为福州个性化问题,又不想影响自家数据库数据,所以另存一个福州小区库,用于手动添加数据 saveOrUpdateFzDist(districts) - remarkRemoveDist(removeDists) - def fzDistricts = getFzDistricts(city) + + if(day == 15){ + def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, "2022-01-01"); + remarkRemoveDist(removeDists) + } + def fzDistricts = null + if(day==15){ + fzDistricts = getFzDistricts(city, null, null) + }else{ + fzDistricts = getFzDistricts(city, startMDate, endMDate) + } for(def rMap : fzDistricts){ //价格没有也提供 @@ -157,7 +176,6 @@ class DistrictInfoExportBatch extends BaseBatch{ udate.put(k,v) } udate.removeField("_id") - udate.put("m_date", new Date()) FZ_DIST_COLL.update(new BasicDBObject('_id',d.get('_id')), new BasicDBObject('$set', udate)); }else{ FZ_DIST_COLL.save(d) @@ -184,9 +202,12 @@ class DistrictInfoExportBatch extends BaseBatch{ } } - def getFzDistricts(def city){ + def getFzDistricts(def city, def mdate, def endMDate){ def q = new BasicDBObject() q.put("city", city) + if(mdate){ + q.put("m_date", new BasicDBObject('$gte', mdate).append('$lt', endMDate)) + } def rets = [] FZ_DIST_COLL.find(q).each{it-> rets << it diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy index 2aa5282a0..c076d0790 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy @@ -24,7 +24,7 @@ import java.util.concurrent.TimeUnit @BatchOwner(Watcher.GaoJuan) class DistrictPriceExportBatch extends BaseBatch{ - def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); +// def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def lastMonthDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); def charset = "GBK" @@ -41,6 +41,7 @@ class DistrictPriceExportBatch extends BaseBatch{ @Override void process(Object args) { + def curDate = Integer.parseInt(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")); def headers = ["id","comm_id","city","district","name","avg_price","year_month"] def ret = createNewDatas("福州") @@ -88,9 +89,13 @@ class DistrictPriceExportBatch extends BaseBatch{ return sb.toString() } - def getFzDistricts(def city){ + + def getFzDistricts(def city, def mdate, def endMDate){ def q = new BasicDBObject() q.put("city", city) + if(mdate){ + q.put("m_date", new BasicDBObject('$gte', mdate).append('$lt', endMDate)) + } def rets = [] FZ_DIST_COLL.find(q).each{it-> rets << it @@ -100,7 +105,17 @@ class DistrictPriceExportBatch extends BaseBatch{ def createNewDatas(def city){ - def districts = getFzDistricts(city) + def districts = null + def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + + def day = new Date().getDate() + def startMDate = DateUtils.truncateDay(new Date()-1) + def endMDate = DateUtils.truncateDay(new Date()) + if(day == 15){ + districts = getFzDistricts(city, null, null) + }else{ + districts = getFzDistricts(city, startMDate, endMDate) + } def curPriceMap = findFzDistrictSellStat("福州",curDate) def result = [] for(def rMap : districts){ @@ -271,9 +286,10 @@ class DistrictPriceExportBatch extends BaseBatch{ } def saveToDB(rets){ + for(def r : rets){ def q = new BasicDBObject() - q.put("month", curDate.toInteger()) + q.put("month", r[6]) q.put("comm_id", r[1]) def obj = FZ_DISTRICT_STAT_COLL.findOne(q) if(obj){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy index cce1de4df..24d6b09fe 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy @@ -70,26 +70,43 @@ class DistrictSellStatBatch extends BaseBatch { def monDate = getMomDate(curMonth) def yoyDate = getYoyDate(curMonth) + + def day = new Date().getDate() + def startMDate = DateUtils.truncateDay(new Date()-1) + def endMDate = DateUtils.truncateDay(new Date()) + def index = 0 for(def city : citys){ index++ def districts = [] if(city == "福州"){ - districts = FZ_DIST_COLL.find(new BasicDBObject("city", city), keys).toList() + if(day == 2){ + districts = FZ_DIST_COLL.find(new BasicDBObject("city", city), keys).toList() + }else{ + districts = FZ_DIST_COLL.find(new BasicDBObject("city", city).append("m_date", new BasicDBObject('$gte',startMDate).append('$lt', endMDate)), keys).toList() + } }else{ - districts = DbUtils.seaweed.find(new BasicDBObject("city", city).append("cat", "district").append("visible", 0), keys).batchSize(2000) + def q = new BasicDBObject("city", city).append("cat", "district").append("status", 0) + if(day == 2){ + districts = DbUtils.seaweed.find(q, keys).batchSize(2000) + }else{ + q.put("m_date", new BasicDBObject('$gte',startMDate).append('$lt', endMDate)) + districts = DbUtils.seaweed.find(q, keys).batchSize(2000) + } } - print city +" "+districts.size() - def weekendPriceMap = findPriceMap(city, lastMonthWeekend)//上个月小区均价 - def curMonthlyPriceMap = findMonthlyPrice(city, curMonth)//当月小区月均价 - def lastMonthlyPriceMap = findMonthlyPrice(city, monDate)//上个月小区月均价 - def lastYearPriceMap = findMonthlyPrice(city, yoyDate)//去年小区月均价 - if(districts && weekendPriceMap){ - running(curMonth, districts, weekendPriceMap, curMonthlyPriceMap, lastMonthlyPriceMap, lastYearPriceMap) + println index+" "+city +" "+districts.size()+" "+ (new Date()).toString() + + if(districts.size()>0){ + def weekendPriceMap = findPriceMap(city, lastMonthWeekend)//上个月小区均价 + def curMonthlyPriceMap = findMonthlyPrice(city, curMonth)//当月小区月均价 + def lastMonthlyPriceMap = findMonthlyPrice(city, monDate)//上个月小区月均价 + def lastYearPriceMap = findMonthlyPrice(city, yoyDate)//去年小区月均价 + if(weekendPriceMap){ + running(curMonth, districts, weekendPriceMap, curMonthlyPriceMap, lastMonthlyPriceMap, lastYearPriceMap) + } } - println index+" "+city +" "+ (new Date()).toString() } } @@ -210,7 +227,7 @@ class DistrictSellStatBatch extends BaseBatch { // }; // }) } - latch.await() +// latch.await() } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy index 67c803131..43fb6f3b3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy @@ -22,11 +22,11 @@ class AddPriceTest { task.run(); } def run(){ - def filePath = "D:\\数据\\建行\\房产估值\\指导价\\202306测试偏差较大案例统计数据——tob价格.xlsx" - addGXDPrice(filePath) + def filePath = "C:\\Users\\gaojuan\\Desktop\\7月成交案例badcase.xlsx" +// addGXDPrice(filePath) // addCcbPrice(filePath) // addThirdPrice(filePath) -// addFjwPrice(filePath) + addFjwPrice(filePath) } @@ -107,11 +107,11 @@ class AddPriceTest { def addGXDPrice(filepath){ ExcelUtils.multiThreadingModify(filepath, { d -> if (d['B']) { - def dist = CommonDatautils.findDistrictById(d['B']) + def dist = CommonDatautils.findDistrictById(d['E']) if(dist){ def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name) // def gxdPrice = findGXDPrice(d['D'], d['E'], d['C']) - d['I'] = gxdPrice + d['L'] = gxdPrice } } }) @@ -136,10 +136,10 @@ class AddPriceTest { def addFjwPrice(path){ ExcelUtils.multiThreadingModify(path,{d-> if(!d['K']){ - def dist = CommonDatautils.findDistrictById(d['B']) + def dist = CommonDatautils.findDistrictById(d['E']) if(dist){ // d['N'] = dist.region - def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2023-06-25","yyyy-MM-dd")) + def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2023-07-16","yyyy-MM-dd")) if(dbo){ // d['K'] = dbo.avg_price d['K'] = dbo.avg_price_2b