新增贝壳找房各城市成交总量统计
This commit is contained in:
+75
@@ -0,0 +1,75 @@
|
||||
package com.fangjia.script.temp.shenjian.crawl.ke
|
||||
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.util.ABuYunUtil
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.jsoup.Jsoup
|
||||
import org.jsoup.nodes.Document
|
||||
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
|
||||
/**
|
||||
* 贝壳各城市成交总量抓取
|
||||
*
|
||||
* @author shenjian
|
||||
* @since 2021/6/2
|
||||
*/
|
||||
class CountTradeKe {
|
||||
|
||||
static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018)
|
||||
static ExecutorService executorService = Executors.newFixedThreadPool(30)
|
||||
|
||||
static void main(String[] args) {
|
||||
crawlTradeCount()
|
||||
//println(count())
|
||||
}
|
||||
|
||||
static crawlTradeCount() {
|
||||
cityConn.find(new BasicDBObject("root", "ke.com"),
|
||||
new BasicDBObject("_id", 1).append("city_short", 1).append("city", 1))
|
||||
.batchSize(100).each { objTemp ->
|
||||
final def obj = objTemp
|
||||
executorService.execute(new Runnable() {
|
||||
@Override
|
||||
void run() {
|
||||
def id = obj._id
|
||||
def city_short = obj.city_short
|
||||
final def URL = "https://" + city_short + ".ke.com/chengjiao/"
|
||||
try {
|
||||
String res = ABuYunUtil.getReponseContent2(URL, "utf-8")
|
||||
if (res) {
|
||||
Document doc = Jsoup.parse(res);
|
||||
boolean existTrade = JQueryUtils.selectText(".resultDes .total", doc).contains("成交房源")
|
||||
if (existTrade) {
|
||||
String str = JQueryUtils.selectText(".resultDes .total span", doc)
|
||||
int count = Integer.parseInt(str)
|
||||
cityConn.update(new BasicDBObject("_id", id), new BasicDBObject('$set', new BasicDBObject("trade_count", count)))
|
||||
} else {
|
||||
cityConn.update(new BasicDBObject("_id", id), new BasicDBObject('$set', new BasicDBObject("trade_count", 0)))
|
||||
}
|
||||
println("当前处理城市" + obj.city)
|
||||
}
|
||||
} catch(Exception ex) {
|
||||
ex.printStackTrace()
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
executorService.shutdown()
|
||||
}
|
||||
|
||||
static count() {
|
||||
int number = 0;
|
||||
cityConn.find(new BasicDBObject("root", "ke.com"), new BasicDBObject("trade_count", 1))
|
||||
.batchSize(100).each { obj ->
|
||||
if (obj.trade_count) {
|
||||
number += obj.trade_count
|
||||
}
|
||||
}
|
||||
return number
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user