Merge branch 'script_gj' of http://gitlab.fangjia.com/dev0/main into script_gj
This commit is contained in:
@@ -287,7 +287,7 @@ public class PoiController {
|
|||||||
double lng = Double.parseDouble(lngStr);
|
double lng = Double.parseDouble(lngStr);
|
||||||
double lat = Double.parseDouble(latStr);
|
double lat = Double.parseDouble(latStr);
|
||||||
Map<String, List<String>> result = new HashMap<String, List<String>>();
|
Map<String, List<String>> result = new HashMap<String, List<String>>();
|
||||||
List<String> bankSmallType = Arrays.asList("银行", "中国人民银行", "国家开发银行", "中国进出口银行",
|
List<String> bankSmallType = Arrays.asList("中国人民银行", "国家开发银行", "中国进出口银行",
|
||||||
"中国银行", "中国工商银行", "中国建设银行", "中国农业银行", "交通银行", "招商银行", "华夏银行",
|
"中国银行", "中国工商银行", "中国建设银行", "中国农业银行", "交通银行", "招商银行", "华夏银行",
|
||||||
"中信银行", "中国民生银行", "中国光大银行", "上海银行", "上海浦东发展银行", "平安银行", "兴业银行",
|
"中信银行", "中国民生银行", "中国光大银行", "上海银行", "上海浦东发展银行", "平安银行", "兴业银行",
|
||||||
"北京银行", "广发银行", "农村商业银行", "香港恒生银行", "东亚银行", "花旗银行", "渣打银行", "汇丰银行",
|
"北京银行", "广发银行", "农村商业银行", "香港恒生银行", "东亚银行", "花旗银行", "渣打银行", "汇丰银行",
|
||||||
|
|||||||
+1
-1
@@ -2617,7 +2617,7 @@ public class PropertyController {
|
|||||||
item.put("city", sqmetreRent.getCity());
|
item.put("city", sqmetreRent.getCity());
|
||||||
item.put("district", sqmetreRent.getRegion());
|
item.put("district", sqmetreRent.getRegion());
|
||||||
item.put("name", sqmetreRent.getDistrictName());
|
item.put("name", sqmetreRent.getDistrictName());
|
||||||
item.put("rent", sqmetreRent.getAvgPrice().intValue());
|
item.put("rent", DataUtil.roundDouble(sqmetreRent.getAvgPrice().doubleValue()/30.0, 2));
|
||||||
item.put("date", DateUtil.formatByPattern(sqmetreRent.getWeekend(), "yyyy-MM"));
|
item.put("date", DateUtil.formatByPattern(sqmetreRent.getWeekend(), "yyyy-MM"));
|
||||||
result.add(item);
|
result.add(item);
|
||||||
}
|
}
|
||||||
|
|||||||
+67
-9
@@ -1,5 +1,6 @@
|
|||||||
package com.fangjia.script.gj.fjwPriceBaseData
|
package com.fangjia.script.gj.fjwPriceBaseData
|
||||||
|
|
||||||
|
import cn.emay.mina.util.ConcurrentHashSet
|
||||||
import com.fangjia.script.BaseBatch
|
import com.fangjia.script.BaseBatch
|
||||||
import com.fangjia.script.BatchOwner
|
import com.fangjia.script.BatchOwner
|
||||||
import com.fangjia.script.Watcher
|
import com.fangjia.script.Watcher
|
||||||
@@ -119,9 +120,14 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
|||||||
def exportSellCSV(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate, houseIdsfilter){
|
def exportSellCSV(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate, houseIdsfilter){
|
||||||
|
|
||||||
def retMap = findAllCommHouse(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate, houseIdsfilter)
|
def retMap = findAllCommHouse(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate, houseIdsfilter)
|
||||||
println city + "(原)房源总量:"+retMap.size()
|
def cityCode = cityCodeDataMap.get(city)
|
||||||
|
def delHouseIds = retMap.del_houses
|
||||||
|
println city + "(删除)房源总量:"+delHouseIds.size()
|
||||||
|
delHousesToPgSql(cityCode, delHouseIds)
|
||||||
|
|
||||||
def allDatas = retMap.values()
|
|
||||||
|
def allDatas = retMap.add_houses.values()
|
||||||
|
println city + "(新增)房源总量:"+allDatas.size()
|
||||||
saveToPgSql(allDatas)
|
saveToPgSql(allDatas)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -133,6 +139,7 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
|||||||
|
|
||||||
BloomFilter<CharSequence> curhousefilter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001);
|
BloomFilter<CharSequence> curhousefilter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001);
|
||||||
def retMap = new ConcurrentHashMap()
|
def retMap = new ConcurrentHashMap()
|
||||||
|
def delHouseIds = new ConcurrentHashSet()
|
||||||
for(def i=0;i<districts.size();i++){
|
for(def i=0;i<districts.size();i++){
|
||||||
final def rMap = districts[i];
|
final def rMap = districts[i];
|
||||||
|
|
||||||
@@ -159,7 +166,10 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
|||||||
def estateType = CcbCodeUtils.parseHouseEstateType(rMap.build_type);
|
def estateType = CcbCodeUtils.parseHouseEstateType(rMap.build_type);
|
||||||
def struCode = CcbCodeUtils.getBuildStructureCode(rMap.build_structure);
|
def struCode = CcbCodeUtils.getBuildStructureCode(rMap.build_structure);
|
||||||
for(def m : ret){
|
for(def m : ret){
|
||||||
if(!houseIdsfilter.mightContain(m.houseId) && !curhousefilter.mightContain(m.houseId)){
|
if(houseIdsfilter.mightContain(m.houseId)){
|
||||||
|
delHouseIds.add(m.houseId);
|
||||||
|
}
|
||||||
|
if(!curhousefilter.mightContain(m.houseId)){ //!houseIdsfilter.mightContain(m.houseId) &&
|
||||||
m.city_code = cityCode
|
m.city_code = cityCode
|
||||||
m.region = areaData.district
|
m.region = areaData.district
|
||||||
m.region_code = regionCode
|
m.region_code = regionCode
|
||||||
@@ -200,7 +210,11 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
latch.await()
|
latch.await()
|
||||||
return retMap
|
|
||||||
|
def results = [:]
|
||||||
|
results.add_houses = retMap
|
||||||
|
results.del_houses = delHouseIds
|
||||||
|
return results
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -487,9 +501,9 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
|||||||
|
|
||||||
|
|
||||||
def findHousesByMdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, rets1){
|
def findHousesByMdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, rets1){
|
||||||
def date = DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd")
|
def date = DateUtils.addMonth(endDate.toString(), -1, "yyyyMMdd")
|
||||||
// def date = DateUtils.addMonth(endDate.toString(), -12, "yyyyMMdd")
|
// def date = DateUtils.addMonth(endDate.toString(), -12, "yyyyMMdd")
|
||||||
def startMdate = DateUtils.parseFromTightByPattern(date,"yyyyMMdd")//DateUtils.parseFromTightByPattern("20240519","yyyyMMdd")
|
def startMdate = DateUtils.parseFromTightByPattern("2025-01-01","yyyyMMdd")//DateUtils.parseFromTightByPattern("20240519","yyyyMMdd")
|
||||||
def endMdate = DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd")//DateUtils.parseFromTightByPattern("20240718", "yyyyMMdd")
|
def endMdate = DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd")//DateUtils.parseFromTightByPattern("20240718", "yyyyMMdd")
|
||||||
|
|
||||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(date.toInteger()).to(endDate).includeLower(false).includeUpper(false);
|
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(date.toInteger()).to(endDate).includeLower(false).includeUpper(false);
|
||||||
@@ -506,9 +520,16 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
|||||||
// .must(area_range).must(dateQueryBuilder)
|
// .must(area_range).must(dateQueryBuilder)
|
||||||
|
|
||||||
|
|
||||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
|
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
|
||||||
.must(date_range).must(area_range).must(date_range_mdate)//.must(QueryBuilders.termQuery("root", "lianjia.com"))
|
.must(QueryBuilders.termQuery("region", region))
|
||||||
|
.must(QueryBuilders.termQuery("cat", cat))
|
||||||
|
.must(area_range)
|
||||||
|
.filter(
|
||||||
|
QueryBuilders.boolQuery()
|
||||||
|
.should(date_range)
|
||||||
|
.should(date_range_mdate)
|
||||||
|
.minimumShouldMatch(1)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
if(!alias){
|
if(!alias){
|
||||||
@@ -968,6 +989,43 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def delHousesToPgSql(cityCode, ids){
|
||||||
|
println "start: "+new Date()
|
||||||
|
|
||||||
|
pgsqlInstance.withTransaction { conn ->
|
||||||
|
def sql = "delete from base.bigdata_forsale where urbn_id=? and gujia_sell_id=?"
|
||||||
|
def stmt = conn.prepareStatement(sql)
|
||||||
|
|
||||||
|
try {
|
||||||
|
int count = 0
|
||||||
|
ids.each { id ->
|
||||||
|
stmt.setObject(1, cityCode)
|
||||||
|
stmt.setObject(2, id)
|
||||||
|
stmt.addBatch()
|
||||||
|
count++
|
||||||
|
|
||||||
|
// 每20000条执行一次
|
||||||
|
if (count % 20000 == 0) {
|
||||||
|
stmt.executeBatch()
|
||||||
|
println "已执行批处理: $count"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 执行剩余的
|
||||||
|
if (ids.size() % 20000 != 0) {
|
||||||
|
stmt.executeBatch()
|
||||||
|
println "执行最后一批: ${ids.size() % 20000}"
|
||||||
|
}
|
||||||
|
|
||||||
|
println "总处理记录数: $count"
|
||||||
|
} finally {
|
||||||
|
stmt.close()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
println "end: "+new Date()
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def saveToPgSql(rows){
|
def saveToPgSql(rows){
|
||||||
for(def r : rows) {
|
for(def r : rows) {
|
||||||
|
|||||||
Reference in New Issue
Block a user