新增贝壳找房抓取异常统计

This commit is contained in:
沈健
2021-06-02 14:36:29 +08:00
parent 3395ae6248
commit 62a419ea66
@@ -20,11 +20,14 @@ import java.util.concurrent.Executors
class CountTradeKe {
static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018)
static DBCollection dealPriceConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
static ExecutorService executorService = Executors.newFixedThreadPool(30)
static void main(String[] args) {
crawlTradeCount()
//crawlTradeCount()
//println(count())
//1896919 1760177
findUnNormalCount()
}
static crawlTradeCount() {
@@ -62,6 +65,28 @@ class CountTradeKe {
executorService.shutdown()
}
/**
* 找出异常抓取的城市
*/
static findUnNormalCount() {
cityConn.find(new BasicDBObject("root", "ke.com"),
new BasicDBObject("_id", 1).append("city", 1).append("city_short", 1).append("trade_count", 1))
.batchSize(100).each { obj ->
Long realTradeCount = obj.trade_count
if (realTradeCount > 0) {
Long crawledCount = dealPriceConn.count(new BasicDBObject("city", obj.city).append("source", "贝壳找房"))
if (realTradeCount > crawledCount) {
println(obj.city + " " + obj.city_short + "抓取异常:抓取为" + crawledCount + " 实际为" + realTradeCount + " 少抓" + (realTradeCount - crawledCount))
}
}
}
}
/**
* 统计实际抓取量
*
* @return
*/
static count() {
int number = 0;
cityConn.find(new BasicDBObject("root", "ke.com"), new BasicDBObject("trade_count", 1))