diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/ke/CountTradeKe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/ke/CountTradeKe.groovy new file mode 100644 index 000000000..064598c6b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/ke/CountTradeKe.groovy @@ -0,0 +1,75 @@ +package com.fangjia.script.temp.shenjian.crawl.ke + +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.jsoup.Jsoup +import org.jsoup.nodes.Document + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 贝壳各城市成交总量抓取 + * + * @author shenjian + * @since 2021/6/2 + */ +class CountTradeKe { + + static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018) + static ExecutorService executorService = Executors.newFixedThreadPool(30) + + static void main(String[] args) { + crawlTradeCount() + //println(count()) + } + + static crawlTradeCount() { + cityConn.find(new BasicDBObject("root", "ke.com"), + new BasicDBObject("_id", 1).append("city_short", 1).append("city", 1)) + .batchSize(100).each { objTemp -> + final def obj = objTemp + executorService.execute(new Runnable() { + @Override + void run() { + def id = obj._id + def city_short = obj.city_short + final def URL = "https://" + city_short + ".ke.com/chengjiao/" + try { + String res = ABuYunUtil.getReponseContent2(URL, "utf-8") + if (res) { + Document doc = Jsoup.parse(res); + boolean existTrade = JQueryUtils.selectText(".resultDes .total", doc).contains("成交房源") + if (existTrade) { + String str = JQueryUtils.selectText(".resultDes .total span", doc) + int count = Integer.parseInt(str) + cityConn.update(new BasicDBObject("_id", id), new BasicDBObject('$set', new BasicDBObject("trade_count", count))) + } else { + cityConn.update(new BasicDBObject("_id", id), new BasicDBObject('$set', new BasicDBObject("trade_count", 0))) + } + println("当前处理城市" + obj.city) + } + } catch(Exception ex) { + ex.printStackTrace() + } + } + }) + } + + executorService.shutdown() + } + + static count() { + int number = 0; + cityConn.find(new BasicDBObject("root", "ke.com"), new BasicDBObject("trade_count", 1)) + .batchSize(100).each { obj -> + if (obj.trade_count) { + number += obj.trade_count + } + } + return number + } +}