新增58小区详情脚本抓取方式

This commit is contained in:
沈健
2021-05-26 17:17:52 +08:00
parent 7fe4a709f5
commit bfbcdd07fa
5 changed files with 112 additions and 54 deletions
@@ -2,6 +2,11 @@ package com.fangjia.script.temp.shenjian
import com.alibaba.fastjson.JSONArray import com.alibaba.fastjson.JSONArray
import com.alibaba.fastjson.JSONObject import com.alibaba.fastjson.JSONObject
import com.baidubce.http.ApiExplorerClient
import com.baidubce.http.AppSigner
import com.baidubce.http.HttpMethodName
import com.baidubce.model.ApiExplorerRequest
import com.baidubce.model.ApiExplorerResponse
import com.fangjia.script.statistics.poi.AmapPoi import com.fangjia.script.statistics.poi.AmapPoi
import com.fangjia.script.temp.shenjian.util.SeaweedUtils import com.fangjia.script.temp.shenjian.util.SeaweedUtils
import org.elasticsearch.geometry.utils.Geohash import org.elasticsearch.geometry.utils.Geohash
@@ -12,11 +17,28 @@ import org.elasticsearch.geometry.utils.Geohash
class Temp { class Temp {
static void main(String[] args) { static void main(String[] args) {
String str = SeaweedUtils.encodeByGeoHash(106.569161, 29.498502) String path = "https://goojia.api.bdymkt.com/estimate";
println(str) ApiExplorerRequest request = new ApiExplorerRequest(HttpMethodName.GET, path);
str = Geohash.stringEncode(106.569161, 29.498502, 10) request.setCredentials("fccf809a0f7748449154af926897aecb", "af1f0088a05c493ab20133ba470f87ef ");
println(str)
// wm5zc4131rmf // 设置header参数
request.addHeaderParameter("Content-Type", "application/json; charset=utf-8");
// 设置query参数
request.addQueryParameter("address", "上海宝山呼玛三村");
request.addQueryParameter("city", "上海");
request.addQueryParameter("size", "100");
ApiExplorerClient client = new ApiExplorerClient(new AppSigner());
try {
ApiExplorerResponse response = client.sendRequest(request);
// 返回结果格式为Json字符串
System.out.println(response.getResult());
} catch (Exception e) {
e.printStackTrace();
}
} }
// 万友七季城C区 经度纬度不同106.569161 29.498502 原始106.56881 29.50058 距离231.7003米 // 万友七季城C区 经度纬度不同106.569161 29.498502 原始106.56881 29.50058 距离231.7003米
@@ -0,0 +1,73 @@
package com.fangjia.script.temp.shenjian.crawl
import com.alibaba.fastjson.JSON
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.temp.shenjian.util.MessageUtils
import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.util.logging.Slf4j
import org.springframework.util.StopWatch
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 58小区详情爬取
*
* @author shenjian* @since 2021/5/26
*/
@Slf4j
class District58DetailCrawl {
static DBCollection districtSeedConn = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018)
static ExecutorService executorService = Executors.newFixedThreadPool(10)
static void main(String[] args) {
AtomicLong count = new AtomicLong()
CountDownLatch latch = new CountDownLatch(100000)
districtSeedConn.find(new BasicDBObject("root", "58.com").append("status", new BasicDBObject('$eq', null)), new BasicDBObject("district_id", 1).append("city_short", 1).append("_id", 1))
.batchSize(1000).limit(100000).each { objTemp ->
def obj = objTemp
executorService.submit(new Runnable() {
@Override
void run() {
try {
final def URL = "https://" + obj.city_short + ".58.com/xiaoqu/" + obj.district_id + "/"
def res = ABuYunUtil.getReponseContent2(URL, "utf-8")
if (res && res.code == 200) {
String content = res.content
if (content.indexOf("访问过于频繁") == -1 && content.indexOf("请输入验证码") == -1) {
if (content.indexOf("您所访问的页面不存在") != -1) {
districtSeedConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "页面不存在")))
} else {
JSON json = new JSONObject()
json.put("project", "district_detail_58")
json.put("url", URL)
JSON result = new JSONObject()
result.put("content", (String) content)
json.put("result", result)
byte[] packedMsg = MessageUtils.pack(json)
CommonUtils.sendMessage("result2parse2dev", packedMsg)
districtSeedConn.update(new BasicDBObject("_id", obj._id),
new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已抓取")))
log.info("已抓取 {}", obj.district_id)
}
}
}
} catch (Exception ex) {
ex.printStackTrace()
} finally {
latch.countDown()
}
}
})
}
}
}
@@ -1,46 +0,0 @@
package com.fangjia.script.temp.shenjian.crawl
import com.alibaba.fastjson.JSON
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.temp.shenjian.util.MessageUtils
import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
/**
* 58小区详情爬取
*
* @author shenjian
* @since 2021/5/26
*/
class DistrictDetailCrawl {
static DBCollection districtSeedConn = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018)
static void main(String[] args) {
districtSeedConn.find(new BasicDBObject("root", "58.com").append("status", new BasicDBObject('$eq', null)), new BasicDBObject("district_id", 1).append("city_short", 1).append("_id", 1))
.batchSize(1000).limit(1000).each { obj ->
final def URL = "https://" + obj.city_short + ".58.com/xiaoqu/" + obj.district_id + "/"
def res = ABuYunUtil.getReponseContent2(URL, "utf-8")
if (res && res.code == 200) {
String content = res.content
JSON json = new JSONObject()
json.put("project", "district_detail_58")
json.put("url", URL)
JSON result = new JSONObject()
result.put("content", (String)content)
json.put("result", result)
byte[] packedMsg = MessageUtils.pack(json)
CommonUtils.sendMessage("result2parse", packedMsg)
districtSeedConn.update(new BasicDBObject("_id", obj._id),
new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已抓取")))
println("已抓取" + obj.district_id)
}
}
}
}
@@ -29,11 +29,10 @@ class SeaweedAroundPoi {
static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017) static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017)
static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi" static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi"
static ExecutorService executorService = Executors.newFixedThreadPool(10) static ExecutorService executorService = Executors.newFixedThreadPool(30)
static void main(String[] args) { static void main(String[] args) {
def types = listAllType() def types = listAllType()
//def types = "0601"
DBObject query = new BasicDBObject() DBObject query = new BasicDBObject()
query.put("status", 1) query.put("status", 1)
conn136.find(query).batchSize(5000).each {objTemp -> conn136.find(query).batchSize(5000).each {objTemp ->
@@ -46,6 +45,10 @@ class SeaweedAroundPoi {
def lat = obj.lat def lat = obj.lat
List<AmapPoi> pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") List<AmapPoi> pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "")
if (CollectionUtils.isNotEmpty(pois)) { if (CollectionUtils.isNotEmpty(pois)) {
if (pois.get(0).id == "10044") {
executorService.shutdown()
throw new Error("当日调用已超限")
}
String jsonStr = JSONObject.toJSONString(pois) String jsonStr = JSONObject.toJSONString(pois)
HttpUtils.postForJson(URL, jsonStr) HttpUtils.postForJson(URL, jsonStr)
conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI"))) conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI")))
@@ -8,6 +8,8 @@ import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.HttpRequestUtil import com.fangjia.script.util.HttpRequestUtil
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import java.util.concurrent.ExecutorService
/** /**
* 高德工具类 * 高德工具类
* *
@@ -138,7 +140,11 @@ class GaoDeUtils {
} else if (json && json.infocode && json.infocode == "10044") { } else if (json && json.infocode && json.infocode == "10044") {
//throw new RuntimeException("开发者的日访问量超限,被系统自动封停,第二天0:00会自动解封") //throw new RuntimeException("开发者的日访问量超限,被系统自动封停,第二天0:00会自动解封")
// 多线程情况下还是exit有效果,抛异常不会停止程序 // 多线程情况下还是exit有效果,抛异常不会停止程序
System.exit(1) //System.exit(1)
AmapPoi amapPoi = new AmapPoi()
amapPoi.id = "10044"
result.add(amapPoi)
break;
} else if (json && json.infocode && json.infocode == "10004") { } else if (json && json.infocode && json.infocode == "10004") {
// 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。
Thread.sleep(60000) Thread.sleep(60000)