新增贝壳找房各城市成交总量统计

This commit is contained in:
沈健
2021-06-02 14:08:19 +08:00
parent ea3ae0cf61
commit 3395ae6248
@@ -0,0 +1,75 @@
package com.fangjia.script.temp.shenjian.crawl.ke
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.jsoup.Jsoup
import org.jsoup.nodes.Document
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
/**
* 贝壳各城市成交总量抓取
*
* @author shenjian
* @since 2021/6/2
*/
class CountTradeKe {
static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018)
static ExecutorService executorService = Executors.newFixedThreadPool(30)
static void main(String[] args) {
crawlTradeCount()
//println(count())
}
static crawlTradeCount() {
cityConn.find(new BasicDBObject("root", "ke.com"),
new BasicDBObject("_id", 1).append("city_short", 1).append("city", 1))
.batchSize(100).each { objTemp ->
final def obj = objTemp
executorService.execute(new Runnable() {
@Override
void run() {
def id = obj._id
def city_short = obj.city_short
final def URL = "https://" + city_short + ".ke.com/chengjiao/"
try {
String res = ABuYunUtil.getReponseContent2(URL, "utf-8")
if (res) {
Document doc = Jsoup.parse(res);
boolean existTrade = JQueryUtils.selectText(".resultDes .total", doc).contains("成交房源")
if (existTrade) {
String str = JQueryUtils.selectText(".resultDes .total span", doc)
int count = Integer.parseInt(str)
cityConn.update(new BasicDBObject("_id", id), new BasicDBObject('$set', new BasicDBObject("trade_count", count)))
} else {
cityConn.update(new BasicDBObject("_id", id), new BasicDBObject('$set', new BasicDBObject("trade_count", 0)))
}
println("当前处理城市" + obj.city)
}
} catch(Exception ex) {
ex.printStackTrace()
}
}
})
}
executorService.shutdown()
}
static count() {
int number = 0;
cityConn.find(new BasicDBObject("root", "ke.com"), new BasicDBObject("trade_count", 1))
.batchSize(100).each { obj ->
if (obj.trade_count) {
number += obj.trade_count
}
}
return number
}
}