From 6c19fafa521e307811403c8d00026199fd46f920 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 28 May 2021 09:08:31 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=AE=89=E5=B1=85=E5=AE=A2?= =?UTF-8?q?=E5=B0=8F=E5=8C=BA=E8=AF=A6=E6=83=85=E8=84=9A=E6=9C=AC=E6=8A=93?= =?UTF-8?q?=E5=8F=96=E6=96=B9=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../anjuke/AnjukeDistrictDetailCrawl.groovy | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/anjuke/AnjukeDistrictDetailCrawl.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/anjuke/AnjukeDistrictDetailCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/anjuke/AnjukeDistrictDetailCrawl.groovy new file mode 100644 index 000000000..c9caccb1a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/anjuke/AnjukeDistrictDetailCrawl.groovy @@ -0,0 +1,75 @@ +package com.fangjia.script.temp.shenjian.crawl.anjuke + +import com.alibaba.fastjson.JSON +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.temp.shenjian.util.MessageUtils +import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import groovy.util.logging.Slf4j + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 安居客小区详情抓取 + * + * @author shenjian + * @since 2021/5/28 + */ +@Slf4j +class AnjukeDistrictDetailCrawl { + + static DBCollection districtSeedConn = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018) + static ExecutorService executorService = Executors.newFixedThreadPool(10) + + static void main(String[] args) { + AtomicLong count = new AtomicLong() + CountDownLatch latch = new CountDownLatch(100000) + districtSeedConn.find(new BasicDBObject("root", "anjuke.com").append("status", new BasicDBObject('$eq', null)), new BasicDBObject("district_id", 1).append("city_short", 1).append("_id", 1)) + .batchSize(1000).limit(1000).each { objTemp -> + def obj = objTemp + executorService.submit(new Runnable() { + @Override + void run() { + try { + final def URL = "https://" + obj.city_short + ".anjuke.com/community/view/" + obj.district_id + "#/" + def res = ABuYunUtil.getReponseContent2(URL, "utf-8") + if (res && res.code == 200) { + String content = res.content + if (content.indexOf("请拖动滑块进行验证") == -1 && content.indexOf("访问验证-安居客") == -1) { + if (content.indexOf("您所访问的页面不存在") != -1) { + districtSeedConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "页面不存在"))) + } else { + JSON json = new JSONObject() + json.put("project", "district_detail_anjuke") + json.put("url", URL) + + JSON result = new JSONObject() + result.put("content", (String) content) + json.put("result", result) + + byte[] packedMsg = MessageUtils.pack(json) + CommonUtils.sendMessage("result2parse2dev", packedMsg) + districtSeedConn.update(new BasicDBObject("_id", obj._id), + new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已抓取"))) + log.info("已抓取 {}", obj.district_id) + } + } + } else if (res && res.code == 508) { + log.info("508 {}", obj.district_id) + } + } catch (Exception ex) { + ex.printStackTrace() + } finally { + latch.countDown() + } + } + }) + } + } +}