diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/ke/CountTradeKe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/ke/CountTradeKe.groovy index 064598c6b..d8b2c466b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/ke/CountTradeKe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/ke/CountTradeKe.groovy @@ -20,11 +20,14 @@ import java.util.concurrent.Executors class CountTradeKe { static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018) + static DBCollection dealPriceConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com") static ExecutorService executorService = Executors.newFixedThreadPool(30) static void main(String[] args) { - crawlTradeCount() + //crawlTradeCount() //println(count()) + //1896919 1760177 + findUnNormalCount() } static crawlTradeCount() { @@ -62,6 +65,28 @@ class CountTradeKe { executorService.shutdown() } + /** + * 找出异常抓取的城市 + */ + static findUnNormalCount() { + cityConn.find(new BasicDBObject("root", "ke.com"), + new BasicDBObject("_id", 1).append("city", 1).append("city_short", 1).append("trade_count", 1)) + .batchSize(100).each { obj -> + Long realTradeCount = obj.trade_count + if (realTradeCount > 0) { + Long crawledCount = dealPriceConn.count(new BasicDBObject("city", obj.city).append("source", "贝壳找房")) + if (realTradeCount > crawledCount) { + println(obj.city + " " + obj.city_short + "抓取异常:抓取为" + crawledCount + " 实际为" + realTradeCount + " 少抓" + (realTradeCount - crawledCount)) + } + } + } + } + + /** + * 统计实际抓取量 + * + * @return + */ static count() { int number = 0; cityConn.find(new BasicDBObject("root", "ke.com"), new BasicDBObject("trade_count", 1))