diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy index c9801ad9e..2c0cd6f32 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.gj.fjwPriceBaseData +import cn.emay.mina.util.ConcurrentHashSet import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher @@ -119,9 +120,14 @@ class FjwBigdataForsaleBatch extends BaseBatch{ def exportSellCSV(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate, houseIdsfilter){ def retMap = findAllCommHouse(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate, houseIdsfilter) - println city + "(原)房源总量:"+retMap.size() + def cityCode = cityCodeDataMap.get(city) + def delHouseIds = retMap.del_houses + println city + "(删除)房源总量:"+delHouseIds.size() + delHousesToPgSql(cityCode, delHouseIds) - def allDatas = retMap.values() + + def allDatas = retMap.add_houses.values() + println city + "(新增)房源总量:"+allDatas.size() saveToPgSql(allDatas) } @@ -133,6 +139,7 @@ class FjwBigdataForsaleBatch extends BaseBatch{ BloomFilter curhousefilter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001); def retMap = new ConcurrentHashMap() + def delHouseIds = new ConcurrentHashSet() for(def i=0;i + def sql = "delete from base.bigdata_forsale where urbn_id=? and gujia_sell_id=?" + def stmt = conn.prepareStatement(sql) + + try { + int count = 0 + ids.each { id -> + stmt.setObject(1, cityCode) + stmt.setObject(2, id) + stmt.addBatch() + count++ + + // 每20000条执行一次 + if (count % 20000 == 0) { + stmt.executeBatch() + println "已执行批处理: $count" + } + } + + // 执行剩余的 + if (ids.size() % 20000 != 0) { + stmt.executeBatch() + println "执行最后一批: ${ids.size() % 20000}" + } + + println "总处理记录数: $count" + } finally { + stmt.close() + } + } + + println "end: "+new Date() + } + def saveToPgSql(rows){ for(def r : rows) {