diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThardParthAnalyBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThardParthAnalyBatch.groovy new file mode 100644 index 000000000..ef6476d48 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThardParthAnalyBatch.groovy @@ -0,0 +1,101 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +@BatchOwner(Watcher.GaoJuan) +class ThardParthAnalyBatch extends BaseBatch{ + + def thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); + static DBCollection THIRD_PARTY_STAT = MongoUtils.getCollectionByName("fangjia_stat", "third_party_stat", "mongo.db.fangjia.com") + + + static main(args) { + runBatch(new ThardParthAnalyBatch(), args) + } + +// static main(args) { +// def task = new ThardParthAnalyBatch() +// task.run() +// } + @Override + void process(Object args) { + run() + } + + def run(){ + def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -2), "yyyy-MM-dd") + def sDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); + println "统计月份:"+sDate+" 查询更新时间:"+mdate + + def crawlTotalCount = findAllEsDataCount(mdate) + println "抓取第三方小区量:"+crawlTotalCount + def synchTotalCount = findAllMongoDatasCount(sDate) + println "同步到第三方库小区量: "+synchTotalCount + def synchHavePriceTotalCount = findAllMongoDatasHavePricesCount(sDate) + println "同步到第三方库有价格数量:"+synchHavePriceTotalCount + def matchTotalCount = findMatchSuccessCount(sDate) + println "第三方小区匹配成功量:"+matchTotalCount + def noMatchTotalCount = findNoMatchCount(sDate) + println "第三方小区匹配还未匹配数量: "+noMatchTotalCount + + saveOrUpdateDatas(sDate, crawlTotalCount, synchTotalCount, synchHavePriceTotalCount, matchTotalCount, noMatchTotalCount); + + } + + + def findAllEsDataCount(mdate){ + def count = null + ElasticUtils.executeSqlQuery("select count(1) as count from community where m_date>='${mdate}'", { d-> + count = d.count + }) + return count; + } + + def findAllMongoDatasCount(s_date){ + def q = new BasicDBObject().append("s_date", s_date) + return thirdPartyColl.count(q) + } + + def findAllMongoDatasHavePricesCount(sDate){ + def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0)) + return thirdPartyColl.count(q) + } + + def findMatchSuccessCount(sDate){ + def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0)).append("status", 4) + return thirdPartyColl.count(q) + } + + def findNoMatchCount(sDate){ + def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0)).append("status", new BasicDBObject('$exists', false)) + return thirdPartyColl.count(q) + } + + def saveOrUpdateDatas(sDate, crawlTotalCount, synchTotalCount, synchHavePriceTotalCount, matchTotalCount, noMatchTotalCount){ + def obj = THIRD_PARTY_STAT.findOne(new BasicDBObject("s_date", sDate)) + + def udata = new BasicDBObject() + udata.put("crawl_total", crawlTotalCount) + udata.put("synch_total", synchTotalCount) + udata.put("synch_have_price_total", synchHavePriceTotalCount) + udata.put("match_total", matchTotalCount) + udata.put("no_match_total", noMatchTotalCount) + udata.put("c_date", new Date()) + if(obj){ + udata.put("m_date", new Date()) + THIRD_PARTY_STAT.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set', udata)) + }else{ + udata.put("s_date", sDate) + THIRD_PARTY_STAT.save(udata) + } + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy index 1645e9131..f5da97fd8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy @@ -921,7 +921,7 @@ class FjwBigdataForsaleBatch extends BaseBatch{ def findAllHouseIdsByCity(){ // 创建布隆过滤器对象 - BloomFilter filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001); + BloomFilter filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 800000000, 0.000001); // def houseIds = new HashSet() pgsqlInstance.eachRow("SELECT gujia_sell_id FROM base.bigdata_forsale",{it-> filter.put(it.gujia_sell_id) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwThirdPartyPriceBatch.groovy index 2d77f4142..daa08b17c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwThirdPartyPriceBatch.groovy @@ -35,9 +35,6 @@ class FjwThirdPartyPriceBatch extends BaseBatch { def sourceMap = ["诸葛找房":"01", "安居客":"02","房天下":"03","链家":"04","城市家居":"05","麦田":"06","贝壳":"07","淘房365":"08","中原":"09","兔博士":"10","Q房":"11","房探":"12", "房多多":"13","我爱我家":"14","禧泰数据":"15","吉屋网":"16","乐居":"17","21世纪":"18","房产超市":"19","58同城":"20"] - def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", - "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") - def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified", "postgres", "Goojia123456", "org.postgresql.Driver") @@ -392,139 +389,6 @@ class FjwThirdPartyPriceBatch extends BaseBatch { return rets; } - def createData3(cityid){ - def sql = "select COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, ZHUGE_AVG_PRICE, ANJUKE_AVG_PRICE, FANG_AVG_PRICE, LIANJIA_AVG_PRICE, CITYHOUSE_AVG_PRICE, MAITIAN_AVG_PRICE, KE_AVG_PRICE, TAOFANG_AVG_PRICE, ZHONGYUAN_AVG_PRICE, TBS_AVG_PRICE, QFANG_AVG_PRICE, FANGTAN_AVG_PRICE, FANGDUODUO_AVG_PRICE, WIWJ_AVG_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from THIRD_PARTY_PRICE_GJ where CITYID='${cityid}' and YEAR_MONTH=${sDate}"; - def allRows = [] - sqlInstance.eachRow(sql,{r-> - def dataMap = [:] - - dataMap.COMMID = r.COMMID - dataMap.COMMNAME = r.COMMNAME - dataMap.CITYID = r.CITYID - dataMap.CITYNAME = r.CITYNAME - dataMap.DISTID = r.DISTID - dataMap.DISTNAME = r.DISTNAME - dataMap.YEAR_MONTH = r.YEAR_MONTH - def price_map = [:] - dataMap.price_map = price_map - if(r.ZHUGE_AVG_PRICE){ - price_map.zhuge_avg_price = r.ZHUGE_AVG_PRICE - } - - if(r.ANJUKE_AVG_PRICE){ - price_map.anjuke_avg_price = r.ANJUKE_AVG_PRICE - } - - if(r.FANG_AVG_PRICE){ - price_map.fang_avg_price = r.FANG_AVG_PRICE - } - - if(r.LIANJIA_AVG_PRICE){ - price_map.lianjia_avg_price = r.LIANJIA_AVG_PRICE - } - - if(r.CITYHOUSE_AVG_PRICE){ - price_map.cityhouse_avg_price = r.CITYHOUSE_AVG_PRICE - } - - if(r.MAITIAN_AVG_PRICE){ - price_map.maitian_avg_price = r.MAITIAN_AVG_PRICE - } - - if(r.KE_AVG_PRICE){ - price_map.ke_avg_price = r.KE_AVG_PRICE - } - - if(r.TAOFANG_AVG_PRICE){ - price_map.taofang_avg_price = r.TAOFANG_AVG_PRICE - } - - if(r.ZHONGYUAN_AVG_PRICE){ - price_map.zhongyuan_avg_price = r.ZHONGYUAN_AVG_PRICE - } - - if(r.TBS_AVG_PRICE){ - price_map.tbs_avg_price = r.TBS_AVG_PRICE - } - - if(r.QFANG_AVG_PRICE){ - price_map.qfang_avg_price = r.QFANG_AVG_PRICE - } - - if(r.FANGTAN_AVG_PRICE){ - price_map.fangtan_avg_price = r.FANGTAN_AVG_PRICE - } - - if(r.FANGDUODUO_AVG_PRICE){ - price_map.fangduoduo_avg_price = r.FANGDUODUO_AVG_PRICE - } - - if(r.WIWJ_AVG_PRICE){ - price_map.wiwj_avg_price = r.WIWJ_AVG_PRICE - } - dataMap.updatetime = r.UPDATETIME - - allRows << dataMap - }); - - - def results = [] - for(def m : allRows){ - def priceMap = m.price_map - priceMap.each{k,v-> - def ret = [] - ret << new ObjectId().toString() - ret << m.CITYID - ret << m.DISTID - ret << m.COMMID - ret << m.COMMNAME - - def web = "" - if(k=="zhuge_avg_price"){ - web = "诸葛找房" - }else if(k=="anjuke_avg_price"){ - web = "安居客" - }else if(k=="fang_avg_price"){ - web = "房天下" - }else if(k=="lianjia_avg_price"){ - web = "链家" - }else if(k=="cityhouse_avg_price"){ - web = "禧泰数据" - }else if(k=="maitian_avg_price"){ - web = "麦田" - }else if(k=="ke_avg_price"){ - web = "贝壳" - }else if(k=="taofang_avg_price"){ - web = "淘房365" - }else if(k=="zhongyuan_avg_price"){ - web = "中原" - }else if(k=="tbs_avg_price"){ - web = "兔博士" - }else if(k=="qfang_avg_price"){ - web = "Q房" - }else if(k=="fangtan_avg_price"){ - web = "房探" - }else if(k=="fangduoduo_avg_price"){ - web = "房多多" - }else if(k=="wiwj_avg_price"){ - web = "我爱我家" - } - - ret << sourceMap.get(web) - ret << web - ret << m.YEAR_MONTH - ret << v - ret << "" - ret << "" - ret << m.updatetime - results << ret - } - } - - return results - - } - def findThirdPartyPrice(def city, def statMap){ def q = new BasicDBObject()