From 90733d071bf0fd5becdecc90746c23fa9f19e0b2 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 7 Mar 2023 14:54:02 +0800 Subject: [PATCH] =?UTF-8?q?=E6=88=BF=E6=BA=90=E5=AF=BC=E5=85=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/BigdataForsaleBatch.groovy | 66 ++++++++++++------- .../ccbevaluation/CcbCommonUtils.groovy | 4 +- 2 files changed, 43 insertions(+), 27 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 82baa96ee..65e5b729d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -80,12 +80,11 @@ class BigdataForsaleBatch extends BaseBatch{ for(def city : citys){ index++ // def distIds = getDistrictIds(cityCodeDataMap.get(city)) + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); - exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230118, 20230219); -// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210901, 20221119); -// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20221011); -// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831); +// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230118, 20230219); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230218, 20230319); //测试通过匹配接口配房源效果 // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118) } @@ -114,8 +113,8 @@ class BigdataForsaleBatch extends BaseBatch{ println city + "(原)房源总量:"+retMap.size() def allDatas = retMap.values() - allDatas = repeatHouseId(allDatas) - println city + "(新)房源总量:"+allDatas.size() +// // allDatas = repeatHouseId(allDatas) +// println city + "(新)房源总量:"+allDatas.size() def fileName = "/home/goojia/ccb/BIGDATA_FORSALE_GJ.dat" @@ -140,6 +139,9 @@ class BigdataForsaleBatch extends BaseBatch{ def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(districts.size()) println city +" 小区量:"+districts.size() + def houseIds = findAllHouseIdsByCity(cityCodeDataMap.get(city)) + println city +" ${city} 库已存在房源:"+districts.size() + def retMap = new ConcurrentHashMap() for(def i=0;i + houseIds << it.FORSALEID + }); + return houseIds; + } + def isExistFromHouse(def houseId){ def sql = "SELECT count(FORSALEID) as count FROM bigdata_forsale_gj WHERE FORSALEID = '"+houseId+"'"; @@ -1109,7 +1125,7 @@ class BigdataForsaleBatch extends BaseBatch{ } } r << null - r << '2023-02-28' + r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28" r << null r << '2999-12-31' r << null diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 2f436c9fd..b0407e1b5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -127,12 +127,12 @@ class CcbCommonUtils { def q = new BasicDBObject() q.put("city", city) q.put('guiding_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(date)).append('$lt', DateUtils.parseFromTightYYYYMMDD(endDate))); - def dbcursor = GUIDING_PRICE_COLL.find(q) + def dbcursor = GUIDING_PRICE_COLL.find(q).sort(new BasicDBObject("guiding_date",-1)) def rets = [] def sets = new HashSet() for(def d : dbcursor){ def guiding_date = DateUtils.formatDateByPattern(d.guiding_date, 'yyyy-MM-dd') - def key = d.city +"##"+d.region+"##"+d.name+"##" + guiding_date + def key = d.city +"##"+d.region+"##"+d.name if(!sets.contains(key)){ rets << d sets << key