新增第三方统计
This commit is contained in:
+101
@@ -0,0 +1,101 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class ThardParthAnalyBatch extends BaseBatch{
|
||||
|
||||
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
static DBCollection THIRD_PARTY_STAT = MongoUtils.getCollectionByName("fangjia_stat", "third_party_stat", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
static main(args) {
|
||||
runBatch(new ThardParthAnalyBatch(), args)
|
||||
}
|
||||
|
||||
// static main(args) {
|
||||
// def task = new ThardParthAnalyBatch()
|
||||
// task.run()
|
||||
// }
|
||||
@Override
|
||||
void process(Object args) {
|
||||
run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -2), "yyyy-MM-dd")
|
||||
def sDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
|
||||
println "统计月份:"+sDate+" 查询更新时间:"+mdate
|
||||
|
||||
def crawlTotalCount = findAllEsDataCount(mdate)
|
||||
println "抓取第三方小区量:"+crawlTotalCount
|
||||
def synchTotalCount = findAllMongoDatasCount(sDate)
|
||||
println "同步到第三方库小区量: "+synchTotalCount
|
||||
def synchHavePriceTotalCount = findAllMongoDatasHavePricesCount(sDate)
|
||||
println "同步到第三方库有价格数量:"+synchHavePriceTotalCount
|
||||
def matchTotalCount = findMatchSuccessCount(sDate)
|
||||
println "第三方小区匹配成功量:"+matchTotalCount
|
||||
def noMatchTotalCount = findNoMatchCount(sDate)
|
||||
println "第三方小区匹配还未匹配数量: "+noMatchTotalCount
|
||||
|
||||
saveOrUpdateDatas(sDate, crawlTotalCount, synchTotalCount, synchHavePriceTotalCount, matchTotalCount, noMatchTotalCount);
|
||||
|
||||
}
|
||||
|
||||
|
||||
def findAllEsDataCount(mdate){
|
||||
def count = null
|
||||
ElasticUtils.executeSqlQuery("select count(1) as count from community where m_date>='${mdate}'", { d->
|
||||
count = d.count
|
||||
})
|
||||
return count;
|
||||
}
|
||||
|
||||
def findAllMongoDatasCount(s_date){
|
||||
def q = new BasicDBObject().append("s_date", s_date)
|
||||
return thirdPartyColl.count(q)
|
||||
}
|
||||
|
||||
def findAllMongoDatasHavePricesCount(sDate){
|
||||
def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0))
|
||||
return thirdPartyColl.count(q)
|
||||
}
|
||||
|
||||
def findMatchSuccessCount(sDate){
|
||||
def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0)).append("status", 4)
|
||||
return thirdPartyColl.count(q)
|
||||
}
|
||||
|
||||
def findNoMatchCount(sDate){
|
||||
def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0)).append("status", new BasicDBObject('$exists', false))
|
||||
return thirdPartyColl.count(q)
|
||||
}
|
||||
|
||||
def saveOrUpdateDatas(sDate, crawlTotalCount, synchTotalCount, synchHavePriceTotalCount, matchTotalCount, noMatchTotalCount){
|
||||
def obj = THIRD_PARTY_STAT.findOne(new BasicDBObject("s_date", sDate))
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("crawl_total", crawlTotalCount)
|
||||
udata.put("synch_total", synchTotalCount)
|
||||
udata.put("synch_have_price_total", synchHavePriceTotalCount)
|
||||
udata.put("match_total", matchTotalCount)
|
||||
udata.put("no_match_total", noMatchTotalCount)
|
||||
udata.put("c_date", new Date())
|
||||
if(obj){
|
||||
udata.put("m_date", new Date())
|
||||
THIRD_PARTY_STAT.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set', udata))
|
||||
}else{
|
||||
udata.put("s_date", sDate)
|
||||
THIRD_PARTY_STAT.save(udata)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
+1
-1
@@ -921,7 +921,7 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
||||
|
||||
def findAllHouseIdsByCity(){
|
||||
// 创建布隆过滤器对象
|
||||
BloomFilter<CharSequence> filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001);
|
||||
BloomFilter<CharSequence> filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 800000000, 0.000001);
|
||||
// def houseIds = new HashSet()
|
||||
pgsqlInstance.eachRow("SELECT gujia_sell_id FROM base.bigdata_forsale",{it->
|
||||
filter.put(it.gujia_sell_id)
|
||||
|
||||
-136
@@ -35,9 +35,6 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
|
||||
def sourceMap = ["诸葛找房":"01", "安居客":"02","房天下":"03","链家":"04","城市家居":"05","麦田":"06","贝壳":"07","淘房365":"08","中原":"09","兔博士":"10","Q房":"11","房探":"12",
|
||||
"房多多":"13","我爱我家":"14","禧泰数据":"15","吉屋网":"16","乐居":"17","21世纪":"18","房产超市":"19","58同城":"20"]
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
|
||||
"postgres", "Goojia123456", "org.postgresql.Driver")
|
||||
@@ -392,139 +389,6 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
|
||||
return rets;
|
||||
}
|
||||
|
||||
def createData3(cityid){
|
||||
def sql = "select COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, ZHUGE_AVG_PRICE, ANJUKE_AVG_PRICE, FANG_AVG_PRICE, LIANJIA_AVG_PRICE, CITYHOUSE_AVG_PRICE, MAITIAN_AVG_PRICE, KE_AVG_PRICE, TAOFANG_AVG_PRICE, ZHONGYUAN_AVG_PRICE, TBS_AVG_PRICE, QFANG_AVG_PRICE, FANGTAN_AVG_PRICE, FANGDUODUO_AVG_PRICE, WIWJ_AVG_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from THIRD_PARTY_PRICE_GJ where CITYID='${cityid}' and YEAR_MONTH=${sDate}";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def dataMap = [:]
|
||||
|
||||
dataMap.COMMID = r.COMMID
|
||||
dataMap.COMMNAME = r.COMMNAME
|
||||
dataMap.CITYID = r.CITYID
|
||||
dataMap.CITYNAME = r.CITYNAME
|
||||
dataMap.DISTID = r.DISTID
|
||||
dataMap.DISTNAME = r.DISTNAME
|
||||
dataMap.YEAR_MONTH = r.YEAR_MONTH
|
||||
def price_map = [:]
|
||||
dataMap.price_map = price_map
|
||||
if(r.ZHUGE_AVG_PRICE){
|
||||
price_map.zhuge_avg_price = r.ZHUGE_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.ANJUKE_AVG_PRICE){
|
||||
price_map.anjuke_avg_price = r.ANJUKE_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.FANG_AVG_PRICE){
|
||||
price_map.fang_avg_price = r.FANG_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.LIANJIA_AVG_PRICE){
|
||||
price_map.lianjia_avg_price = r.LIANJIA_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.CITYHOUSE_AVG_PRICE){
|
||||
price_map.cityhouse_avg_price = r.CITYHOUSE_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.MAITIAN_AVG_PRICE){
|
||||
price_map.maitian_avg_price = r.MAITIAN_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.KE_AVG_PRICE){
|
||||
price_map.ke_avg_price = r.KE_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.TAOFANG_AVG_PRICE){
|
||||
price_map.taofang_avg_price = r.TAOFANG_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.ZHONGYUAN_AVG_PRICE){
|
||||
price_map.zhongyuan_avg_price = r.ZHONGYUAN_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.TBS_AVG_PRICE){
|
||||
price_map.tbs_avg_price = r.TBS_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.QFANG_AVG_PRICE){
|
||||
price_map.qfang_avg_price = r.QFANG_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.FANGTAN_AVG_PRICE){
|
||||
price_map.fangtan_avg_price = r.FANGTAN_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.FANGDUODUO_AVG_PRICE){
|
||||
price_map.fangduoduo_avg_price = r.FANGDUODUO_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.WIWJ_AVG_PRICE){
|
||||
price_map.wiwj_avg_price = r.WIWJ_AVG_PRICE
|
||||
}
|
||||
dataMap.updatetime = r.UPDATETIME
|
||||
|
||||
allRows << dataMap
|
||||
});
|
||||
|
||||
|
||||
def results = []
|
||||
for(def m : allRows){
|
||||
def priceMap = m.price_map
|
||||
priceMap.each{k,v->
|
||||
def ret = []
|
||||
ret << new ObjectId().toString()
|
||||
ret << m.CITYID
|
||||
ret << m.DISTID
|
||||
ret << m.COMMID
|
||||
ret << m.COMMNAME
|
||||
|
||||
def web = ""
|
||||
if(k=="zhuge_avg_price"){
|
||||
web = "诸葛找房"
|
||||
}else if(k=="anjuke_avg_price"){
|
||||
web = "安居客"
|
||||
}else if(k=="fang_avg_price"){
|
||||
web = "房天下"
|
||||
}else if(k=="lianjia_avg_price"){
|
||||
web = "链家"
|
||||
}else if(k=="cityhouse_avg_price"){
|
||||
web = "禧泰数据"
|
||||
}else if(k=="maitian_avg_price"){
|
||||
web = "麦田"
|
||||
}else if(k=="ke_avg_price"){
|
||||
web = "贝壳"
|
||||
}else if(k=="taofang_avg_price"){
|
||||
web = "淘房365"
|
||||
}else if(k=="zhongyuan_avg_price"){
|
||||
web = "中原"
|
||||
}else if(k=="tbs_avg_price"){
|
||||
web = "兔博士"
|
||||
}else if(k=="qfang_avg_price"){
|
||||
web = "Q房"
|
||||
}else if(k=="fangtan_avg_price"){
|
||||
web = "房探"
|
||||
}else if(k=="fangduoduo_avg_price"){
|
||||
web = "房多多"
|
||||
}else if(k=="wiwj_avg_price"){
|
||||
web = "我爱我家"
|
||||
}
|
||||
|
||||
ret << sourceMap.get(web)
|
||||
ret << web
|
||||
ret << m.YEAR_MONTH
|
||||
ret << v
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << m.updatetime
|
||||
results << ret
|
||||
}
|
||||
}
|
||||
|
||||
return results
|
||||
|
||||
}
|
||||
|
||||
|
||||
def findThirdPartyPrice(def city, def statMap){
|
||||
def q = new BasicDBObject()
|
||||
|
||||
Reference in New Issue
Block a user