同步房源修改,对于后面价格更改的房源进行删除新增

This commit is contained in:
2025-12-19 18:03:36 +08:00
parent a1154e786f
commit 6cf26b835e
@@ -1,5 +1,6 @@
package com.fangjia.script.gj.fjwPriceBaseData
import cn.emay.mina.util.ConcurrentHashSet
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
@@ -119,9 +120,14 @@ class FjwBigdataForsaleBatch extends BaseBatch{
def exportSellCSV(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate, houseIdsfilter){
def retMap = findAllCommHouse(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate, houseIdsfilter)
println city + "(原)房源总量:"+retMap.size()
def cityCode = cityCodeDataMap.get(city)
def delHouseIds = retMap.del_houses
println city + "(删除)房源总量:"+delHouseIds.size()
delHousesToPgSql(cityCode, delHouseIds)
def allDatas = retMap.values()
def allDatas = retMap.add_houses.values()
println city + "(新增)房源总量:"+allDatas.size()
saveToPgSql(allDatas)
}
@@ -133,6 +139,7 @@ class FjwBigdataForsaleBatch extends BaseBatch{
BloomFilter<CharSequence> curhousefilter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001);
def retMap = new ConcurrentHashMap()
def delHouseIds = new ConcurrentHashSet()
for(def i=0;i<districts.size();i++){
final def rMap = districts[i];
@@ -159,7 +166,10 @@ class FjwBigdataForsaleBatch extends BaseBatch{
def estateType = CcbCodeUtils.parseHouseEstateType(rMap.build_type);
def struCode = CcbCodeUtils.getBuildStructureCode(rMap.build_structure);
for(def m : ret){
if(!houseIdsfilter.mightContain(m.houseId) && !curhousefilter.mightContain(m.houseId)){
if(houseIdsfilter.mightContain(m.houseId)){
delHouseIds.add(m.houseId);
}
if(!curhousefilter.mightContain(m.houseId)){ //!houseIdsfilter.mightContain(m.houseId) &&
m.city_code = cityCode
m.region = areaData.district
m.region_code = regionCode
@@ -200,7 +210,11 @@ class FjwBigdataForsaleBatch extends BaseBatch{
})
}
latch.await()
return retMap
def results = [:]
results.add_houses = retMap
results.del_houses = delHouseIds
return results
}
@@ -487,9 +501,9 @@ class FjwBigdataForsaleBatch extends BaseBatch{
def findHousesByMdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, rets1){
def date = DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd")
def date = DateUtils.addMonth(endDate.toString(), -1, "yyyyMMdd")
// def date = DateUtils.addMonth(endDate.toString(), -12, "yyyyMMdd")
def startMdate = DateUtils.parseFromTightByPattern(date,"yyyyMMdd")//DateUtils.parseFromTightByPattern("20240519","yyyyMMdd")
def startMdate = DateUtils.parseFromTightByPattern("2025-01-01","yyyyMMdd")//DateUtils.parseFromTightByPattern("20240519","yyyyMMdd")
def endMdate = DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd")//DateUtils.parseFromTightByPattern("20240718", "yyyyMMdd")
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(date.toInteger()).to(endDate).includeLower(false).includeUpper(false);
@@ -506,9 +520,16 @@ class FjwBigdataForsaleBatch extends BaseBatch{
// .must(area_range).must(dateQueryBuilder)
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
.must(date_range).must(area_range).must(date_range_mdate)//.must(QueryBuilders.termQuery("root", "lianjia.com"))
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
.must(QueryBuilders.termQuery("region", region))
.must(QueryBuilders.termQuery("cat", cat))
.must(area_range)
.filter(
QueryBuilders.boolQuery()
.should(date_range)
.should(date_range_mdate)
.minimumShouldMatch(1)
)
if(!alias){
@@ -968,6 +989,43 @@ class FjwBigdataForsaleBatch extends BaseBatch{
}
def delHousesToPgSql(cityCode, ids){
println "start: "+new Date()
pgsqlInstance.withTransaction { conn ->
def sql = "delete from base.bigdata_forsale where urbn_id=? and gujia_sell_id=?"
def stmt = conn.prepareStatement(sql)
try {
int count = 0
ids.each { id ->
stmt.setObject(1, cityCode)
stmt.setObject(2, id)
stmt.addBatch()
count++
// 每20000条执行一次
if (count % 20000 == 0) {
stmt.executeBatch()
println "已执行批处理: $count"
}
}
// 执行剩余的
if (ids.size() % 20000 != 0) {
stmt.executeBatch()
println "执行最后一批: ${ids.size() % 20000}"
}
println "总处理记录数: $count"
} finally {
stmt.close()
}
}
println "end: "+new Date()
}
def saveToPgSql(rows){
for(def r : rows) {