Merge branch 'script_gj' into 'master'

update

See merge request !690
This commit is contained in:
2023-07-20 17:22:59 +08:00
@@ -37,7 +37,7 @@ class BigdataDealBatch extends BaseBatch{
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd")); def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"] def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"]
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017"; public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
@@ -83,17 +83,18 @@ class BigdataDealBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def ppeDists = [:] def ppeDists = [:]
def dealIds = findHistoryDealIds(city)
def statMap = [:]//getDistrictStatByCity(city, weekend) def statMap = [:]//getDistrictStatByCity(city, weekend)
if(city=="上海"){ if(city=="上海"){
def youDataTrade = getYouDaData(statMap) def youDataTrade = getYouDaData(statMap)
def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda") def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda", dealIds)
rows.addAll(youdaDatas) rows.addAll(youdaDatas)
println city +" "+youdaDatas.size() println city +" "+youdaDatas.size()
} }
def trades = getCityData(city, statMap, ppeDists) def trades = getCityData(city, statMap, ppeDists)
if(trades){ if(trades){
def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") def rets = createData(trades, cityCodeDataMap, distCodeMap, "other", dealIds)
rows.addAll(rets) rows.addAll(rets)
println "【${index}】"+city +" 成交共:"+rets.size() println "【${index}】"+city +" 成交共:"+rets.size()
@@ -115,7 +116,7 @@ class BigdataDealBatch extends BaseBatch{
saveToPgSql(rows) saveToPgSql(rows)
} }
def createData(def rets, cityCodeDatas, distCodeMap, source){ def createData(def rets, cityCodeDatas, distCodeMap, source, dealIds){
def result = Collections.synchronizedList(new ArrayList()); def result = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong() def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size()) final CountDownLatch latch = new CountDownLatch(rets.size())
@@ -127,6 +128,10 @@ class BigdataDealBatch extends BaseBatch{
def ccbPriceMap = getCommunityPrices("上海") def ccbPriceMap = getCommunityPrices("上海")
for(def i=0;i<rets.size();i++){ for(def i=0;i<rets.size();i++){
final def trade = rets[i] final def trade = rets[i]
if(dealIds.contains(trade._id.toString())){
println trade._id.toString()
continue
}
fixedThreadPool.execute(new Runnable() { fixedThreadPool.execute(new Runnable() {
void run() { void run() {
try { try {
@@ -623,6 +628,20 @@ class BigdataDealBatch extends BaseBatch{
return map return map
} }
def findHistoryDealIds(def city){
def sql = "SELECT gujia_bds_mdl_id from base.t0831_bigdata_deal_gj_h where urbn_nm='${city}' "
def ids = []
try{
pgsqlInstance.eachRow(sql,{
ids << it.gujia_bds_mdl_id
});
}catch(Exception e){
}
return ids
}
def getPpeGTE15Datas(def city){ def getPpeGTE15Datas(def city){
def sql = "select unit_price,commid from avm_evaluation_item_gj where data_type='成交' and pe>=15 and pe<=20 " + def sql = "select unit_price,commid from avm_evaluation_item_gj where data_type='成交' and pe>=15 and pe<=20 " +
"and cityname='"+city+"'"; "and cityname='"+city+"'";