新增第三方统计

This commit is contained in:
2024-08-09 14:52:02 +08:00
parent e20fef3455
commit 29414e98b0
3 changed files with 102 additions and 137 deletions
@@ -0,0 +1,101 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
@BatchOwner(Watcher.GaoJuan)
class ThardParthAnalyBatch extends BaseBatch{
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
static DBCollection THIRD_PARTY_STAT = MongoUtils.getCollectionByName("fangjia_stat", "third_party_stat", "mongo.db.fangjia.com")
static main(args) {
runBatch(new ThardParthAnalyBatch(), args)
}
// static main(args) {
// def task = new ThardParthAnalyBatch()
// task.run()
// }
@Override
void process(Object args) {
run()
}
def run(){
def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -2), "yyyy-MM-dd")
def sDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
println "统计月份:"+sDate+" 查询更新时间:"+mdate
def crawlTotalCount = findAllEsDataCount(mdate)
println "抓取第三方小区量:"+crawlTotalCount
def synchTotalCount = findAllMongoDatasCount(sDate)
println "同步到第三方库小区量: "+synchTotalCount
def synchHavePriceTotalCount = findAllMongoDatasHavePricesCount(sDate)
println "同步到第三方库有价格数量:"+synchHavePriceTotalCount
def matchTotalCount = findMatchSuccessCount(sDate)
println "第三方小区匹配成功量:"+matchTotalCount
def noMatchTotalCount = findNoMatchCount(sDate)
println "第三方小区匹配还未匹配数量: "+noMatchTotalCount
saveOrUpdateDatas(sDate, crawlTotalCount, synchTotalCount, synchHavePriceTotalCount, matchTotalCount, noMatchTotalCount);
}
def findAllEsDataCount(mdate){
def count = null
ElasticUtils.executeSqlQuery("select count(1) as count from community where m_date>='${mdate}'", { d->
count = d.count
})
return count;
}
def findAllMongoDatasCount(s_date){
def q = new BasicDBObject().append("s_date", s_date)
return thirdPartyColl.count(q)
}
def findAllMongoDatasHavePricesCount(sDate){
def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0))
return thirdPartyColl.count(q)
}
def findMatchSuccessCount(sDate){
def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0)).append("status", 4)
return thirdPartyColl.count(q)
}
def findNoMatchCount(sDate){
def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0)).append("status", new BasicDBObject('$exists', false))
return thirdPartyColl.count(q)
}
def saveOrUpdateDatas(sDate, crawlTotalCount, synchTotalCount, synchHavePriceTotalCount, matchTotalCount, noMatchTotalCount){
def obj = THIRD_PARTY_STAT.findOne(new BasicDBObject("s_date", sDate))
def udata = new BasicDBObject()
udata.put("crawl_total", crawlTotalCount)
udata.put("synch_total", synchTotalCount)
udata.put("synch_have_price_total", synchHavePriceTotalCount)
udata.put("match_total", matchTotalCount)
udata.put("no_match_total", noMatchTotalCount)
udata.put("c_date", new Date())
if(obj){
udata.put("m_date", new Date())
THIRD_PARTY_STAT.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set', udata))
}else{
udata.put("s_date", sDate)
THIRD_PARTY_STAT.save(udata)
}
}
}
@@ -921,7 +921,7 @@ class FjwBigdataForsaleBatch extends BaseBatch{
def findAllHouseIdsByCity(){
// 创建布隆过滤器对象
BloomFilter<CharSequence> filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001);
BloomFilter<CharSequence> filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 800000000, 0.000001);
// def houseIds = new HashSet()
pgsqlInstance.eachRow("SELECT gujia_sell_id FROM base.bigdata_forsale",{it->
filter.put(it.gujia_sell_id)
@@ -35,9 +35,6 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
def sourceMap = ["诸葛找房":"01", "安居客":"02","房天下":"03","链家":"04","城市家居":"05","麦田":"06","贝壳":"07","淘房365":"08","中原":"09","兔博士":"10","Q房":"11","房探":"12",
"房多多":"13","我爱我家":"14","禧泰数据":"15","吉屋网":"16","乐居":"17","21世纪":"18","房产超市":"19","58同城":"20"]
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
@@ -392,139 +389,6 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
return rets;
}
def createData3(cityid){
def sql = "select COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, ZHUGE_AVG_PRICE, ANJUKE_AVG_PRICE, FANG_AVG_PRICE, LIANJIA_AVG_PRICE, CITYHOUSE_AVG_PRICE, MAITIAN_AVG_PRICE, KE_AVG_PRICE, TAOFANG_AVG_PRICE, ZHONGYUAN_AVG_PRICE, TBS_AVG_PRICE, QFANG_AVG_PRICE, FANGTAN_AVG_PRICE, FANGDUODUO_AVG_PRICE, WIWJ_AVG_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from THIRD_PARTY_PRICE_GJ where CITYID='${cityid}' and YEAR_MONTH=${sDate}";
def allRows = []
sqlInstance.eachRow(sql,{r->
def dataMap = [:]
dataMap.COMMID = r.COMMID
dataMap.COMMNAME = r.COMMNAME
dataMap.CITYID = r.CITYID
dataMap.CITYNAME = r.CITYNAME
dataMap.DISTID = r.DISTID
dataMap.DISTNAME = r.DISTNAME
dataMap.YEAR_MONTH = r.YEAR_MONTH
def price_map = [:]
dataMap.price_map = price_map
if(r.ZHUGE_AVG_PRICE){
price_map.zhuge_avg_price = r.ZHUGE_AVG_PRICE
}
if(r.ANJUKE_AVG_PRICE){
price_map.anjuke_avg_price = r.ANJUKE_AVG_PRICE
}
if(r.FANG_AVG_PRICE){
price_map.fang_avg_price = r.FANG_AVG_PRICE
}
if(r.LIANJIA_AVG_PRICE){
price_map.lianjia_avg_price = r.LIANJIA_AVG_PRICE
}
if(r.CITYHOUSE_AVG_PRICE){
price_map.cityhouse_avg_price = r.CITYHOUSE_AVG_PRICE
}
if(r.MAITIAN_AVG_PRICE){
price_map.maitian_avg_price = r.MAITIAN_AVG_PRICE
}
if(r.KE_AVG_PRICE){
price_map.ke_avg_price = r.KE_AVG_PRICE
}
if(r.TAOFANG_AVG_PRICE){
price_map.taofang_avg_price = r.TAOFANG_AVG_PRICE
}
if(r.ZHONGYUAN_AVG_PRICE){
price_map.zhongyuan_avg_price = r.ZHONGYUAN_AVG_PRICE
}
if(r.TBS_AVG_PRICE){
price_map.tbs_avg_price = r.TBS_AVG_PRICE
}
if(r.QFANG_AVG_PRICE){
price_map.qfang_avg_price = r.QFANG_AVG_PRICE
}
if(r.FANGTAN_AVG_PRICE){
price_map.fangtan_avg_price = r.FANGTAN_AVG_PRICE
}
if(r.FANGDUODUO_AVG_PRICE){
price_map.fangduoduo_avg_price = r.FANGDUODUO_AVG_PRICE
}
if(r.WIWJ_AVG_PRICE){
price_map.wiwj_avg_price = r.WIWJ_AVG_PRICE
}
dataMap.updatetime = r.UPDATETIME
allRows << dataMap
});
def results = []
for(def m : allRows){
def priceMap = m.price_map
priceMap.each{k,v->
def ret = []
ret << new ObjectId().toString()
ret << m.CITYID
ret << m.DISTID
ret << m.COMMID
ret << m.COMMNAME
def web = ""
if(k=="zhuge_avg_price"){
web = "诸葛找房"
}else if(k=="anjuke_avg_price"){
web = "安居客"
}else if(k=="fang_avg_price"){
web = "房天下"
}else if(k=="lianjia_avg_price"){
web = "链家"
}else if(k=="cityhouse_avg_price"){
web = "禧泰数据"
}else if(k=="maitian_avg_price"){
web = "麦田"
}else if(k=="ke_avg_price"){
web = "贝壳"
}else if(k=="taofang_avg_price"){
web = "淘房365"
}else if(k=="zhongyuan_avg_price"){
web = "中原"
}else if(k=="tbs_avg_price"){
web = "兔博士"
}else if(k=="qfang_avg_price"){
web = "Q房"
}else if(k=="fangtan_avg_price"){
web = "房探"
}else if(k=="fangduoduo_avg_price"){
web = "房多多"
}else if(k=="wiwj_avg_price"){
web = "我爱我家"
}
ret << sourceMap.get(web)
ret << web
ret << m.YEAR_MONTH
ret << v
ret << ""
ret << ""
ret << m.updatetime
results << ret
}
}
return results
}
def findThirdPartyPrice(def city, def statMap){
def q = new BasicDBObject()