diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/District58DetailCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/District58DetailCrawl.groovy new file mode 100644 index 000000000..ca7a1b7bd --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/District58DetailCrawl.groovy @@ -0,0 +1,67 @@ +package com.fangjia.script.temp.shenjian.crawl + +import com.alibaba.fastjson.JSON +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.temp.shenjian.util.MessageUtils +import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import groovy.util.logging.Slf4j +import org.springframework.util.StopWatch + +/** + * 58小区详情爬取 + * + * @author shenjian + * @since 2021/5/26 + */ +@Slf4j +class District58DetailCrawl { + + static DBCollection districtSeedConn = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018) + + static void main(String[] args) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start("District58DetailCrawl") + long count = 1; + districtSeedConn.find(new BasicDBObject("root", "58.com").append("status", new BasicDBObject('$eq', null)), new BasicDBObject("district_id", 1).append("city_short", 1).append("_id", 1)) + .batchSize(1000).limit(1000).each { obj -> + final def URL = "https://" + obj.city_short + ".58.com/xiaoqu/" + obj.district_id + "/" + def res = ABuYunUtil.getReponseContent2(URL, "utf-8") + if (res && res.code == 200) { + String content = res.content + if (content.indexOf("访问过于频繁") == -1 && content.indexOf("请输入验证码") == -1) { + if (content.indexOf("您所访问的页面不存在") != -1) { + districtSeedConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "页面不存在"))) + } else { + JSON json = new JSONObject() + json.put("project", "district_detail_58") + json.put("url", URL) + + JSON result = new JSONObject() + result.put("content", (String)content) + json.put("result", result) + + byte[] packedMsg = MessageUtils.pack(json) + CommonUtils.sendMessage("result2parse2dev", packedMsg) + districtSeedConn.update(new BasicDBObject("_id", obj._id), + new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已抓取"))) + if (count % 20 == 0) { + stopWatch.stop() + log.info("抓取总量为{}, 耗时{}", count, stopWatch.getTotalTimeSeconds() / 60) + stopWatch.start() + } + count++ + log.info("已抓取 {}", obj.district_id) + } + } + } + } + if (stopWatch.isRunning()) { + stopWatch.stop() + log.info("抓取完毕。。。抓取总量为{}, 耗时{}", count++, stopWatch.getTotalTimeSeconds() / 60) + } + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/DistrictDetailCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/DistrictDetailCrawl.groovy deleted file mode 100644 index 095456eed..000000000 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/DistrictDetailCrawl.groovy +++ /dev/null @@ -1,46 +0,0 @@ -package com.fangjia.script.temp.shenjian.crawl - -import com.alibaba.fastjson.JSON -import com.alibaba.fastjson.JSONObject -import com.fangjia.script.temp.shenjian.util.MessageUtils -import com.fangjia.script.util.ABuYunUtil -import com.fangjia.script.util.CommonUtils -import com.fangjia.script.util.MongoUtils -import com.mongodb.BasicDBObject -import com.mongodb.DBCollection - -/** - * 58小区详情爬取 - * - * @author shenjian - * @since 2021/5/26 - */ -class DistrictDetailCrawl { - - static DBCollection districtSeedConn = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018) - - static void main(String[] args) { - districtSeedConn.find(new BasicDBObject("root", "58.com").append("status", new BasicDBObject('$eq', null)), new BasicDBObject("district_id", 1).append("city_short", 1).append("_id", 1)) - .batchSize(1000).limit(1000).each { obj -> - final def URL = "https://" + obj.city_short + ".58.com/xiaoqu/" + obj.district_id + "/" - def res = ABuYunUtil.getReponseContent2(URL, "utf-8") - if (res && res.code == 200) { - String content = res.content - - JSON json = new JSONObject() - json.put("project", "district_detail_58") - json.put("url", URL) - - JSON result = new JSONObject() - result.put("content", (String)content) - json.put("result", result) - - byte[] packedMsg = MessageUtils.pack(json) - CommonUtils.sendMessage("result2parse", packedMsg) - districtSeedConn.update(new BasicDBObject("_id", obj._id), - new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已抓取"))) - println("已抓取" + obj.district_id) - } - } - } -} \ No newline at end of file