diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/anjuke/AnjukeDistrictDetailCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/anjuke/AnjukeDistrictDetailCrawl.groovy new file mode 100644 index 000000000..c9caccb1a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/anjuke/AnjukeDistrictDetailCrawl.groovy @@ -0,0 +1,75 @@ +package com.fangjia.script.temp.shenjian.crawl.anjuke + +import com.alibaba.fastjson.JSON +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.temp.shenjian.util.MessageUtils +import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import groovy.util.logging.Slf4j + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 安居客小区详情抓取 + * + * @author shenjian + * @since 2021/5/28 + */ +@Slf4j +class AnjukeDistrictDetailCrawl { + + static DBCollection districtSeedConn = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018) + static ExecutorService executorService = Executors.newFixedThreadPool(10) + + static void main(String[] args) { + AtomicLong count = new AtomicLong() + CountDownLatch latch = new CountDownLatch(100000) + districtSeedConn.find(new BasicDBObject("root", "anjuke.com").append("status", new BasicDBObject('$eq', null)), new BasicDBObject("district_id", 1).append("city_short", 1).append("_id", 1)) + .batchSize(1000).limit(1000).each { objTemp -> + def obj = objTemp + executorService.submit(new Runnable() { + @Override + void run() { + try { + final def URL = "https://" + obj.city_short + ".anjuke.com/community/view/" + obj.district_id + "#/" + def res = ABuYunUtil.getReponseContent2(URL, "utf-8") + if (res && res.code == 200) { + String content = res.content + if (content.indexOf("请拖动滑块进行验证") == -1 && content.indexOf("访问验证-安居客") == -1) { + if (content.indexOf("您所访问的页面不存在") != -1) { + districtSeedConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "页面不存在"))) + } else { + JSON json = new JSONObject() + json.put("project", "district_detail_anjuke") + json.put("url", URL) + + JSON result = new JSONObject() + result.put("content", (String) content) + json.put("result", result) + + byte[] packedMsg = MessageUtils.pack(json) + CommonUtils.sendMessage("result2parse2dev", packedMsg) + districtSeedConn.update(new BasicDBObject("_id", obj._id), + new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已抓取"))) + log.info("已抓取 {}", obj.district_id) + } + } + } else if (res && res.code == 508) { + log.info("508 {}", obj.district_id) + } + } catch (Exception ex) { + ex.printStackTrace() + } finally { + latch.countDown() + } + } + }) + } + } +}