From 019a18b28e30cc0a4bd6990b3c097238cd1bcc60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 26 May 2021 17:17:52 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E58=E5=B0=8F=E5=8C=BA=E8=AF=A6?= =?UTF-8?q?=E6=83=85=E8=84=9A=E6=9C=AC=E6=8A=93=E5=8F=96=E6=96=B9=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../crawl/58/District58DetailCrawl.groovy | 67 +++++++++++++++++++ .../crawl/58/DistrictDetailCrawl.groovy | 46 ------------- 2 files changed, 67 insertions(+), 46 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/District58DetailCrawl.groovy delete mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/DistrictDetailCrawl.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/District58DetailCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/District58DetailCrawl.groovy new file mode 100644 index 000000000..ca7a1b7bd --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/District58DetailCrawl.groovy @@ -0,0 +1,67 @@ +package com.fangjia.script.temp.shenjian.crawl + +import com.alibaba.fastjson.JSON +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.temp.shenjian.util.MessageUtils +import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import groovy.util.logging.Slf4j +import org.springframework.util.StopWatch + +/** + * 58小区详情爬取 + * + * @author shenjian + * @since 2021/5/26 + */ +@Slf4j +class District58DetailCrawl { + + static DBCollection districtSeedConn = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018) + + static void main(String[] args) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start("District58DetailCrawl") + long count = 1; + districtSeedConn.find(new BasicDBObject("root", "58.com").append("status", new BasicDBObject('$eq', null)), new BasicDBObject("district_id", 1).append("city_short", 1).append("_id", 1)) + .batchSize(1000).limit(1000).each { obj -> + final def URL = "https://" + obj.city_short + ".58.com/xiaoqu/" + obj.district_id + "/" + def res = ABuYunUtil.getReponseContent2(URL, "utf-8") + if (res && res.code == 200) { + String content = res.content + if (content.indexOf("访问过于频繁") == -1 && content.indexOf("请输入验证码") == -1) { + if (content.indexOf("您所访问的页面不存在") != -1) { + districtSeedConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "页面不存在"))) + } else { + JSON json = new JSONObject() + json.put("project", "district_detail_58") + json.put("url", URL) + + JSON result = new JSONObject() + result.put("content", (String)content) + json.put("result", result) + + byte[] packedMsg = MessageUtils.pack(json) + CommonUtils.sendMessage("result2parse2dev", packedMsg) + districtSeedConn.update(new BasicDBObject("_id", obj._id), + new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已抓取"))) + if (count % 20 == 0) { + stopWatch.stop() + log.info("抓取总量为{}, 耗时{}", count, stopWatch.getTotalTimeSeconds() / 60) + stopWatch.start() + } + count++ + log.info("已抓取 {}", obj.district_id) + } + } + } + } + if (stopWatch.isRunning()) { + stopWatch.stop() + log.info("抓取完毕。。。抓取总量为{}, 耗时{}", count++, stopWatch.getTotalTimeSeconds() / 60) + } + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/DistrictDetailCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/DistrictDetailCrawl.groovy deleted file mode 100644 index 095456eed..000000000 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/DistrictDetailCrawl.groovy +++ /dev/null @@ -1,46 +0,0 @@ -package com.fangjia.script.temp.shenjian.crawl - -import com.alibaba.fastjson.JSON -import com.alibaba.fastjson.JSONObject -import com.fangjia.script.temp.shenjian.util.MessageUtils -import com.fangjia.script.util.ABuYunUtil -import com.fangjia.script.util.CommonUtils -import com.fangjia.script.util.MongoUtils -import com.mongodb.BasicDBObject -import com.mongodb.DBCollection - -/** - * 58小区详情爬取 - * - * @author shenjian - * @since 2021/5/26 - */ -class DistrictDetailCrawl { - - static DBCollection districtSeedConn = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018) - - static void main(String[] args) { - districtSeedConn.find(new BasicDBObject("root", "58.com").append("status", new BasicDBObject('$eq', null)), new BasicDBObject("district_id", 1).append("city_short", 1).append("_id", 1)) - .batchSize(1000).limit(1000).each { obj -> - final def URL = "https://" + obj.city_short + ".58.com/xiaoqu/" + obj.district_id + "/" - def res = ABuYunUtil.getReponseContent2(URL, "utf-8") - if (res && res.code == 200) { - String content = res.content - - JSON json = new JSONObject() - json.put("project", "district_detail_58") - json.put("url", URL) - - JSON result = new JSONObject() - result.put("content", (String)content) - json.put("result", result) - - byte[] packedMsg = MessageUtils.pack(json) - CommonUtils.sendMessage("result2parse", packedMsg) - districtSeedConn.update(new BasicDBObject("_id", obj._id), - new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已抓取"))) - println("已抓取" + obj.district_id) - } - } - } -} \ No newline at end of file