From b92d6eb9a9c0457e7a2185fbf922376e8b00bf63 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 20 Jul 2023 17:18:22 +0800 Subject: [PATCH] update --- .../ccbevaluation/BigdataDealBatch.groovy | 27 ++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 667458d29..03efd7a79 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -37,7 +37,7 @@ class BigdataDealBatch extends BaseBatch{ def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd")); - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"] public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017"; @@ -83,17 +83,18 @@ class BigdataDealBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def ppeDists = [:] + def dealIds = findHistoryDealIds(city) def statMap = [:]//getDistrictStatByCity(city, weekend) if(city=="上海"){ def youDataTrade = getYouDaData(statMap) - def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda") + def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda", dealIds) rows.addAll(youdaDatas) println city +" "+youdaDatas.size() } def trades = getCityData(city, statMap, ppeDists) if(trades){ - def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") + def rets = createData(trades, cityCodeDataMap, distCodeMap, "other", dealIds) rows.addAll(rets) println "【${index}】"+city +" 成交共:"+rets.size() @@ -115,7 +116,7 @@ class BigdataDealBatch extends BaseBatch{ saveToPgSql(rows) } - def createData(def rets, cityCodeDatas, distCodeMap, source){ + def createData(def rets, cityCodeDatas, distCodeMap, source, dealIds){ def result = Collections.synchronizedList(new ArrayList()); def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(rets.size()) @@ -127,6 +128,10 @@ class BigdataDealBatch extends BaseBatch{ def ccbPriceMap = getCommunityPrices("上海") for(def i=0;i