新增安居客小区详情脚本抓取方式
This commit is contained in:
+75
@@ -0,0 +1,75 @@
|
||||
package com.fangjia.script.temp.shenjian.crawl.anjuke
|
||||
|
||||
import com.alibaba.fastjson.JSON
|
||||
import com.alibaba.fastjson.JSONObject
|
||||
import com.fangjia.script.temp.shenjian.util.MessageUtils
|
||||
import com.fangjia.script.util.ABuYunUtil
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import groovy.util.logging.Slf4j
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/**
|
||||
* 安居客小区详情抓取
|
||||
*
|
||||
* @author shenjian
|
||||
* @since 2021/5/28
|
||||
*/
|
||||
@Slf4j
|
||||
class AnjukeDistrictDetailCrawl {
|
||||
|
||||
static DBCollection districtSeedConn = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018)
|
||||
static ExecutorService executorService = Executors.newFixedThreadPool(10)
|
||||
|
||||
static void main(String[] args) {
|
||||
AtomicLong count = new AtomicLong()
|
||||
CountDownLatch latch = new CountDownLatch(100000)
|
||||
districtSeedConn.find(new BasicDBObject("root", "anjuke.com").append("status", new BasicDBObject('$eq', null)), new BasicDBObject("district_id", 1).append("city_short", 1).append("_id", 1))
|
||||
.batchSize(1000).limit(1000).each { objTemp ->
|
||||
def obj = objTemp
|
||||
executorService.submit(new Runnable() {
|
||||
@Override
|
||||
void run() {
|
||||
try {
|
||||
final def URL = "https://" + obj.city_short + ".anjuke.com/community/view/" + obj.district_id + "#/"
|
||||
def res = ABuYunUtil.getReponseContent2(URL, "utf-8")
|
||||
if (res && res.code == 200) {
|
||||
String content = res.content
|
||||
if (content.indexOf("请拖动滑块进行验证") == -1 && content.indexOf("访问验证-安居客") == -1) {
|
||||
if (content.indexOf("您所访问的页面不存在") != -1) {
|
||||
districtSeedConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "页面不存在")))
|
||||
} else {
|
||||
JSON json = new JSONObject()
|
||||
json.put("project", "district_detail_anjuke")
|
||||
json.put("url", URL)
|
||||
|
||||
JSON result = new JSONObject()
|
||||
result.put("content", (String) content)
|
||||
json.put("result", result)
|
||||
|
||||
byte[] packedMsg = MessageUtils.pack(json)
|
||||
CommonUtils.sendMessage("result2parse2dev", packedMsg)
|
||||
districtSeedConn.update(new BasicDBObject("_id", obj._id),
|
||||
new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已抓取")))
|
||||
log.info("已抓取 {}", obj.district_id)
|
||||
}
|
||||
}
|
||||
} else if (res && res.code == 508) {
|
||||
log.info("508 {}", obj.district_id)
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
ex.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user