新增58小区详情脚本抓取方式

This commit is contained in:
沈健
2021-05-26 17:17:52 +08:00
parent 7fe4a709f5
commit 019a18b28e
2 changed files with 67 additions and 46 deletions
@@ -0,0 +1,67 @@
package com.fangjia.script.temp.shenjian.crawl
import com.alibaba.fastjson.JSON
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.temp.shenjian.util.MessageUtils
import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.util.logging.Slf4j
import org.springframework.util.StopWatch
/**
* 58小区详情爬取
*
* @author shenjian
* @since 2021/5/26
*/
@Slf4j
class District58DetailCrawl {
static DBCollection districtSeedConn = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018)
static void main(String[] args) {
StopWatch stopWatch = new StopWatch();
stopWatch.start("District58DetailCrawl")
long count = 1;
districtSeedConn.find(new BasicDBObject("root", "58.com").append("status", new BasicDBObject('$eq', null)), new BasicDBObject("district_id", 1).append("city_short", 1).append("_id", 1))
.batchSize(1000).limit(1000).each { obj ->
final def URL = "https://" + obj.city_short + ".58.com/xiaoqu/" + obj.district_id + "/"
def res = ABuYunUtil.getReponseContent2(URL, "utf-8")
if (res && res.code == 200) {
String content = res.content
if (content.indexOf("访问过于频繁") == -1 && content.indexOf("请输入验证码") == -1) {
if (content.indexOf("您所访问的页面不存在") != -1) {
districtSeedConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "页面不存在")))
} else {
JSON json = new JSONObject()
json.put("project", "district_detail_58")
json.put("url", URL)
JSON result = new JSONObject()
result.put("content", (String)content)
json.put("result", result)
byte[] packedMsg = MessageUtils.pack(json)
CommonUtils.sendMessage("result2parse2dev", packedMsg)
districtSeedConn.update(new BasicDBObject("_id", obj._id),
new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已抓取")))
if (count % 20 == 0) {
stopWatch.stop()
log.info("抓取总量为{}, 耗时{}", count, stopWatch.getTotalTimeSeconds() / 60)
stopWatch.start()
}
count++
log.info("已抓取 {}", obj.district_id)
}
}
}
}
if (stopWatch.isRunning()) {
stopWatch.stop()
log.info("抓取完毕。。。抓取总量为{}, 耗时{}", count++, stopWatch.getTotalTimeSeconds() / 60)
}
}
}
@@ -1,46 +0,0 @@
package com.fangjia.script.temp.shenjian.crawl
import com.alibaba.fastjson.JSON
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.temp.shenjian.util.MessageUtils
import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
/**
* 58小区详情爬取
*
* @author shenjian
* @since 2021/5/26
*/
class DistrictDetailCrawl {
static DBCollection districtSeedConn = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018)
static void main(String[] args) {
districtSeedConn.find(new BasicDBObject("root", "58.com").append("status", new BasicDBObject('$eq', null)), new BasicDBObject("district_id", 1).append("city_short", 1).append("_id", 1))
.batchSize(1000).limit(1000).each { obj ->
final def URL = "https://" + obj.city_short + ".58.com/xiaoqu/" + obj.district_id + "/"
def res = ABuYunUtil.getReponseContent2(URL, "utf-8")
if (res && res.code == 200) {
String content = res.content
JSON json = new JSONObject()
json.put("project", "district_detail_58")
json.put("url", URL)
JSON result = new JSONObject()
result.put("content", (String)content)
json.put("result", result)
byte[] packedMsg = MessageUtils.pack(json)
CommonUtils.sendMessage("result2parse", packedMsg)
districtSeedConn.update(new BasicDBObject("_id", obj._id),
new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已抓取")))
println("已抓取" + obj.district_id)
}
}
}
}