From 62a419ea669e6bbe34c88fe2510c3244c40aea90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 2 Jun 2021 14:36:29 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E8=B4=9D=E5=A3=B3=E6=89=BE?= =?UTF-8?q?=E6=88=BF=E6=8A=93=E5=8F=96=E5=BC=82=E5=B8=B8=E7=BB=9F=E8=AE=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/crawl/ke/CountTradeKe.groovy | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/ke/CountTradeKe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/ke/CountTradeKe.groovy index 064598c6b..d8b2c466b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/ke/CountTradeKe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/ke/CountTradeKe.groovy @@ -20,11 +20,14 @@ import java.util.concurrent.Executors class CountTradeKe { static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018) + static DBCollection dealPriceConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com") static ExecutorService executorService = Executors.newFixedThreadPool(30) static void main(String[] args) { - crawlTradeCount() + //crawlTradeCount() //println(count()) + //1896919 1760177 + findUnNormalCount() } static crawlTradeCount() { @@ -62,6 +65,28 @@ class CountTradeKe { executorService.shutdown() } + /** + * 找出异常抓取的城市 + */ + static findUnNormalCount() { + cityConn.find(new BasicDBObject("root", "ke.com"), + new BasicDBObject("_id", 1).append("city", 1).append("city_short", 1).append("trade_count", 1)) + .batchSize(100).each { obj -> + Long realTradeCount = obj.trade_count + if (realTradeCount > 0) { + Long crawledCount = dealPriceConn.count(new BasicDBObject("city", obj.city).append("source", "贝壳找房")) + if (realTradeCount > crawledCount) { + println(obj.city + " " + obj.city_short + "抓取异常:抓取为" + crawledCount + " 实际为" + realTradeCount + " 少抓" + (realTradeCount - crawledCount)) + } + } + } + } + + /** + * 统计实际抓取量 + * + * @return + */ static count() { int number = 0; cityConn.find(new BasicDBObject("root", "ke.com"), new BasicDBObject("trade_count", 1))