新增贝壳找房抓取异常统计
This commit is contained in:
+26
-1
@@ -20,11 +20,14 @@ import java.util.concurrent.Executors
|
||||
class CountTradeKe {
|
||||
|
||||
static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018)
|
||||
static DBCollection dealPriceConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
|
||||
static ExecutorService executorService = Executors.newFixedThreadPool(30)
|
||||
|
||||
static void main(String[] args) {
|
||||
crawlTradeCount()
|
||||
//crawlTradeCount()
|
||||
//println(count())
|
||||
//1896919 1760177
|
||||
findUnNormalCount()
|
||||
}
|
||||
|
||||
static crawlTradeCount() {
|
||||
@@ -62,6 +65,28 @@ class CountTradeKe {
|
||||
executorService.shutdown()
|
||||
}
|
||||
|
||||
/**
|
||||
* 找出异常抓取的城市
|
||||
*/
|
||||
static findUnNormalCount() {
|
||||
cityConn.find(new BasicDBObject("root", "ke.com"),
|
||||
new BasicDBObject("_id", 1).append("city", 1).append("city_short", 1).append("trade_count", 1))
|
||||
.batchSize(100).each { obj ->
|
||||
Long realTradeCount = obj.trade_count
|
||||
if (realTradeCount > 0) {
|
||||
Long crawledCount = dealPriceConn.count(new BasicDBObject("city", obj.city).append("source", "贝壳找房"))
|
||||
if (realTradeCount > crawledCount) {
|
||||
println(obj.city + " " + obj.city_short + "抓取异常:抓取为" + crawledCount + " 实际为" + realTradeCount + " 少抓" + (realTradeCount - crawledCount))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 统计实际抓取量
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
static count() {
|
||||
int number = 0;
|
||||
cityConn.find(new BasicDBObject("root", "ke.com"), new BasicDBObject("trade_count", 1))
|
||||
|
||||
Reference in New Issue
Block a user