同步房源修改,对于后面价格更改的房源进行删除新增
This commit is contained in:
+67
-9
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.script.gj.fjwPriceBaseData
|
||||
|
||||
import cn.emay.mina.util.ConcurrentHashSet
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
@@ -119,9 +120,14 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
||||
def exportSellCSV(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate, houseIdsfilter){
|
||||
|
||||
def retMap = findAllCommHouse(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate, houseIdsfilter)
|
||||
println city + "(原)房源总量:"+retMap.size()
|
||||
def cityCode = cityCodeDataMap.get(city)
|
||||
def delHouseIds = retMap.del_houses
|
||||
println city + "(删除)房源总量:"+delHouseIds.size()
|
||||
delHousesToPgSql(cityCode, delHouseIds)
|
||||
|
||||
def allDatas = retMap.values()
|
||||
|
||||
def allDatas = retMap.add_houses.values()
|
||||
println city + "(新增)房源总量:"+allDatas.size()
|
||||
saveToPgSql(allDatas)
|
||||
}
|
||||
|
||||
@@ -133,6 +139,7 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
||||
|
||||
BloomFilter<CharSequence> curhousefilter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001);
|
||||
def retMap = new ConcurrentHashMap()
|
||||
def delHouseIds = new ConcurrentHashSet()
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def rMap = districts[i];
|
||||
|
||||
@@ -159,7 +166,10 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
||||
def estateType = CcbCodeUtils.parseHouseEstateType(rMap.build_type);
|
||||
def struCode = CcbCodeUtils.getBuildStructureCode(rMap.build_structure);
|
||||
for(def m : ret){
|
||||
if(!houseIdsfilter.mightContain(m.houseId) && !curhousefilter.mightContain(m.houseId)){
|
||||
if(houseIdsfilter.mightContain(m.houseId)){
|
||||
delHouseIds.add(m.houseId);
|
||||
}
|
||||
if(!curhousefilter.mightContain(m.houseId)){ //!houseIdsfilter.mightContain(m.houseId) &&
|
||||
m.city_code = cityCode
|
||||
m.region = areaData.district
|
||||
m.region_code = regionCode
|
||||
@@ -200,7 +210,11 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
return retMap
|
||||
|
||||
def results = [:]
|
||||
results.add_houses = retMap
|
||||
results.del_houses = delHouseIds
|
||||
return results
|
||||
}
|
||||
|
||||
|
||||
@@ -487,9 +501,9 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
||||
|
||||
|
||||
def findHousesByMdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, rets1){
|
||||
def date = DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd")
|
||||
def date = DateUtils.addMonth(endDate.toString(), -1, "yyyyMMdd")
|
||||
// def date = DateUtils.addMonth(endDate.toString(), -12, "yyyyMMdd")
|
||||
def startMdate = DateUtils.parseFromTightByPattern(date,"yyyyMMdd")//DateUtils.parseFromTightByPattern("20240519","yyyyMMdd")
|
||||
def startMdate = DateUtils.parseFromTightByPattern("2025-01-01","yyyyMMdd")//DateUtils.parseFromTightByPattern("20240519","yyyyMMdd")
|
||||
def endMdate = DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd")//DateUtils.parseFromTightByPattern("20240718", "yyyyMMdd")
|
||||
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(date.toInteger()).to(endDate).includeLower(false).includeUpper(false);
|
||||
@@ -506,9 +520,16 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
||||
// .must(area_range).must(dateQueryBuilder)
|
||||
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
|
||||
.must(date_range).must(area_range).must(date_range_mdate)//.must(QueryBuilders.termQuery("root", "lianjia.com"))
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
|
||||
.must(QueryBuilders.termQuery("region", region))
|
||||
.must(QueryBuilders.termQuery("cat", cat))
|
||||
.must(area_range)
|
||||
.filter(
|
||||
QueryBuilders.boolQuery()
|
||||
.should(date_range)
|
||||
.should(date_range_mdate)
|
||||
.minimumShouldMatch(1)
|
||||
)
|
||||
|
||||
|
||||
if(!alias){
|
||||
@@ -968,6 +989,43 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
|
||||
def delHousesToPgSql(cityCode, ids){
|
||||
println "start: "+new Date()
|
||||
|
||||
pgsqlInstance.withTransaction { conn ->
|
||||
def sql = "delete from base.bigdata_forsale where urbn_id=? and gujia_sell_id=?"
|
||||
def stmt = conn.prepareStatement(sql)
|
||||
|
||||
try {
|
||||
int count = 0
|
||||
ids.each { id ->
|
||||
stmt.setObject(1, cityCode)
|
||||
stmt.setObject(2, id)
|
||||
stmt.addBatch()
|
||||
count++
|
||||
|
||||
// 每20000条执行一次
|
||||
if (count % 20000 == 0) {
|
||||
stmt.executeBatch()
|
||||
println "已执行批处理: $count"
|
||||
}
|
||||
}
|
||||
|
||||
// 执行剩余的
|
||||
if (ids.size() % 20000 != 0) {
|
||||
stmt.executeBatch()
|
||||
println "执行最后一批: ${ids.size() % 20000}"
|
||||
}
|
||||
|
||||
println "总处理记录数: $count"
|
||||
} finally {
|
||||
stmt.close()
|
||||
}
|
||||
}
|
||||
|
||||
println "end: "+new Date()
|
||||
}
|
||||
|
||||
|
||||
def saveToPgSql(rows){
|
||||
for(def r : rows) {
|
||||
|
||||
Reference in New Issue
Block a user