新增58小区详情脚本抓取方式
This commit is contained in:
+27
-5
@@ -2,6 +2,11 @@ package com.fangjia.script.temp.shenjian
|
|||||||
|
|
||||||
import com.alibaba.fastjson.JSONArray
|
import com.alibaba.fastjson.JSONArray
|
||||||
import com.alibaba.fastjson.JSONObject
|
import com.alibaba.fastjson.JSONObject
|
||||||
|
import com.baidubce.http.ApiExplorerClient
|
||||||
|
import com.baidubce.http.AppSigner
|
||||||
|
import com.baidubce.http.HttpMethodName
|
||||||
|
import com.baidubce.model.ApiExplorerRequest
|
||||||
|
import com.baidubce.model.ApiExplorerResponse
|
||||||
import com.fangjia.script.statistics.poi.AmapPoi
|
import com.fangjia.script.statistics.poi.AmapPoi
|
||||||
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
|
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
|
||||||
import org.elasticsearch.geometry.utils.Geohash
|
import org.elasticsearch.geometry.utils.Geohash
|
||||||
@@ -12,11 +17,28 @@ import org.elasticsearch.geometry.utils.Geohash
|
|||||||
class Temp {
|
class Temp {
|
||||||
|
|
||||||
static void main(String[] args) {
|
static void main(String[] args) {
|
||||||
String str = SeaweedUtils.encodeByGeoHash(106.569161, 29.498502)
|
String path = "https://goojia.api.bdymkt.com/estimate";
|
||||||
println(str)
|
ApiExplorerRequest request = new ApiExplorerRequest(HttpMethodName.GET, path);
|
||||||
str = Geohash.stringEncode(106.569161, 29.498502, 10)
|
request.setCredentials("fccf809a0f7748449154af926897aecb", "af1f0088a05c493ab20133ba470f87ef ");
|
||||||
println(str)
|
|
||||||
// wm5zc4131rmf
|
// 设置header参数
|
||||||
|
request.addHeaderParameter("Content-Type", "application/json; charset=utf-8");
|
||||||
|
|
||||||
|
// 设置query参数
|
||||||
|
request.addQueryParameter("address", "上海宝山呼玛三村");
|
||||||
|
request.addQueryParameter("city", "上海");
|
||||||
|
request.addQueryParameter("size", "100");
|
||||||
|
|
||||||
|
|
||||||
|
ApiExplorerClient client = new ApiExplorerClient(new AppSigner());
|
||||||
|
|
||||||
|
try {
|
||||||
|
ApiExplorerResponse response = client.sendRequest(request);
|
||||||
|
// 返回结果格式为Json字符串
|
||||||
|
System.out.println(response.getResult());
|
||||||
|
} catch (Exception e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 万友七季城C区 经度纬度不同106.569161 29.498502 原始106.56881 29.50058 距离231.7003米
|
// 万友七季城C区 经度纬度不同106.569161 29.498502 原始106.56881 29.50058 距离231.7003米
|
||||||
|
|||||||
+73
@@ -0,0 +1,73 @@
|
|||||||
|
package com.fangjia.script.temp.shenjian.crawl
|
||||||
|
|
||||||
|
import com.alibaba.fastjson.JSON
|
||||||
|
import com.alibaba.fastjson.JSONObject
|
||||||
|
import com.fangjia.script.temp.shenjian.util.MessageUtils
|
||||||
|
import com.fangjia.script.util.ABuYunUtil
|
||||||
|
import com.fangjia.script.util.CommonUtils
|
||||||
|
import com.fangjia.script.util.MongoUtils
|
||||||
|
import com.mongodb.BasicDBObject
|
||||||
|
import com.mongodb.DBCollection
|
||||||
|
import groovy.util.logging.Slf4j
|
||||||
|
import org.springframework.util.StopWatch
|
||||||
|
|
||||||
|
import java.util.concurrent.CountDownLatch
|
||||||
|
import java.util.concurrent.ExecutorService
|
||||||
|
import java.util.concurrent.Executors
|
||||||
|
import java.util.concurrent.atomic.AtomicLong
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 58小区详情爬取
|
||||||
|
*
|
||||||
|
* @author shenjian* @since 2021/5/26
|
||||||
|
*/
|
||||||
|
@Slf4j
|
||||||
|
class District58DetailCrawl {
|
||||||
|
|
||||||
|
static DBCollection districtSeedConn = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018)
|
||||||
|
static ExecutorService executorService = Executors.newFixedThreadPool(10)
|
||||||
|
|
||||||
|
static void main(String[] args) {
|
||||||
|
AtomicLong count = new AtomicLong()
|
||||||
|
CountDownLatch latch = new CountDownLatch(100000)
|
||||||
|
districtSeedConn.find(new BasicDBObject("root", "58.com").append("status", new BasicDBObject('$eq', null)), new BasicDBObject("district_id", 1).append("city_short", 1).append("_id", 1))
|
||||||
|
.batchSize(1000).limit(100000).each { objTemp ->
|
||||||
|
def obj = objTemp
|
||||||
|
executorService.submit(new Runnable() {
|
||||||
|
@Override
|
||||||
|
void run() {
|
||||||
|
try {
|
||||||
|
final def URL = "https://" + obj.city_short + ".58.com/xiaoqu/" + obj.district_id + "/"
|
||||||
|
def res = ABuYunUtil.getReponseContent2(URL, "utf-8")
|
||||||
|
if (res && res.code == 200) {
|
||||||
|
String content = res.content
|
||||||
|
if (content.indexOf("访问过于频繁") == -1 && content.indexOf("请输入验证码") == -1) {
|
||||||
|
if (content.indexOf("您所访问的页面不存在") != -1) {
|
||||||
|
districtSeedConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "页面不存在")))
|
||||||
|
} else {
|
||||||
|
JSON json = new JSONObject()
|
||||||
|
json.put("project", "district_detail_58")
|
||||||
|
json.put("url", URL)
|
||||||
|
|
||||||
|
JSON result = new JSONObject()
|
||||||
|
result.put("content", (String) content)
|
||||||
|
json.put("result", result)
|
||||||
|
|
||||||
|
byte[] packedMsg = MessageUtils.pack(json)
|
||||||
|
CommonUtils.sendMessage("result2parse2dev", packedMsg)
|
||||||
|
districtSeedConn.update(new BasicDBObject("_id", obj._id),
|
||||||
|
new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已抓取")))
|
||||||
|
log.info("已抓取 {}", obj.district_id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (Exception ex) {
|
||||||
|
ex.printStackTrace()
|
||||||
|
} finally {
|
||||||
|
latch.countDown()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
-46
@@ -1,46 +0,0 @@
|
|||||||
package com.fangjia.script.temp.shenjian.crawl
|
|
||||||
|
|
||||||
import com.alibaba.fastjson.JSON
|
|
||||||
import com.alibaba.fastjson.JSONObject
|
|
||||||
import com.fangjia.script.temp.shenjian.util.MessageUtils
|
|
||||||
import com.fangjia.script.util.ABuYunUtil
|
|
||||||
import com.fangjia.script.util.CommonUtils
|
|
||||||
import com.fangjia.script.util.MongoUtils
|
|
||||||
import com.mongodb.BasicDBObject
|
|
||||||
import com.mongodb.DBCollection
|
|
||||||
|
|
||||||
/**
|
|
||||||
* 58小区详情爬取
|
|
||||||
*
|
|
||||||
* @author shenjian
|
|
||||||
* @since 2021/5/26
|
|
||||||
*/
|
|
||||||
class DistrictDetailCrawl {
|
|
||||||
|
|
||||||
static DBCollection districtSeedConn = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018)
|
|
||||||
|
|
||||||
static void main(String[] args) {
|
|
||||||
districtSeedConn.find(new BasicDBObject("root", "58.com").append("status", new BasicDBObject('$eq', null)), new BasicDBObject("district_id", 1).append("city_short", 1).append("_id", 1))
|
|
||||||
.batchSize(1000).limit(1000).each { obj ->
|
|
||||||
final def URL = "https://" + obj.city_short + ".58.com/xiaoqu/" + obj.district_id + "/"
|
|
||||||
def res = ABuYunUtil.getReponseContent2(URL, "utf-8")
|
|
||||||
if (res && res.code == 200) {
|
|
||||||
String content = res.content
|
|
||||||
|
|
||||||
JSON json = new JSONObject()
|
|
||||||
json.put("project", "district_detail_58")
|
|
||||||
json.put("url", URL)
|
|
||||||
|
|
||||||
JSON result = new JSONObject()
|
|
||||||
result.put("content", (String)content)
|
|
||||||
json.put("result", result)
|
|
||||||
|
|
||||||
byte[] packedMsg = MessageUtils.pack(json)
|
|
||||||
CommonUtils.sendMessage("result2parse", packedMsg)
|
|
||||||
districtSeedConn.update(new BasicDBObject("_id", obj._id),
|
|
||||||
new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已抓取")))
|
|
||||||
println("已抓取" + obj.district_id)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+5
-2
@@ -29,11 +29,10 @@ class SeaweedAroundPoi {
|
|||||||
|
|
||||||
static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017)
|
static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017)
|
||||||
static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi"
|
static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi"
|
||||||
static ExecutorService executorService = Executors.newFixedThreadPool(10)
|
static ExecutorService executorService = Executors.newFixedThreadPool(30)
|
||||||
|
|
||||||
static void main(String[] args) {
|
static void main(String[] args) {
|
||||||
def types = listAllType()
|
def types = listAllType()
|
||||||
//def types = "0601"
|
|
||||||
DBObject query = new BasicDBObject()
|
DBObject query = new BasicDBObject()
|
||||||
query.put("status", 1)
|
query.put("status", 1)
|
||||||
conn136.find(query).batchSize(5000).each {objTemp ->
|
conn136.find(query).batchSize(5000).each {objTemp ->
|
||||||
@@ -46,6 +45,10 @@ class SeaweedAroundPoi {
|
|||||||
def lat = obj.lat
|
def lat = obj.lat
|
||||||
List<AmapPoi> pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "")
|
List<AmapPoi> pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "")
|
||||||
if (CollectionUtils.isNotEmpty(pois)) {
|
if (CollectionUtils.isNotEmpty(pois)) {
|
||||||
|
if (pois.get(0).id == "10044") {
|
||||||
|
executorService.shutdown()
|
||||||
|
throw new Error("当日调用已超限")
|
||||||
|
}
|
||||||
String jsonStr = JSONObject.toJSONString(pois)
|
String jsonStr = JSONObject.toJSONString(pois)
|
||||||
HttpUtils.postForJson(URL, jsonStr)
|
HttpUtils.postForJson(URL, jsonStr)
|
||||||
conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI")))
|
conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI")))
|
||||||
|
|||||||
+7
-1
@@ -8,6 +8,8 @@ import com.fangjia.script.util.GeoHelper
|
|||||||
import com.fangjia.script.util.HttpRequestUtil
|
import com.fangjia.script.util.HttpRequestUtil
|
||||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||||
|
|
||||||
|
import java.util.concurrent.ExecutorService
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 高德工具类
|
* 高德工具类
|
||||||
*
|
*
|
||||||
@@ -138,7 +140,11 @@ class GaoDeUtils {
|
|||||||
} else if (json && json.infocode && json.infocode == "10044") {
|
} else if (json && json.infocode && json.infocode == "10044") {
|
||||||
//throw new RuntimeException("开发者的日访问量超限,被系统自动封停,第二天0:00会自动解封")
|
//throw new RuntimeException("开发者的日访问量超限,被系统自动封停,第二天0:00会自动解封")
|
||||||
// 多线程情况下还是exit有效果,抛异常不会停止程序
|
// 多线程情况下还是exit有效果,抛异常不会停止程序
|
||||||
System.exit(1)
|
//System.exit(1)
|
||||||
|
AmapPoi amapPoi = new AmapPoi()
|
||||||
|
amapPoi.id = "10044"
|
||||||
|
result.add(amapPoi)
|
||||||
|
break;
|
||||||
} else if (json && json.infocode && json.infocode == "10004") {
|
} else if (json && json.infocode && json.infocode == "10004") {
|
||||||
// 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。
|
// 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。
|
||||||
Thread.sleep(60000)
|
Thread.sleep(60000)
|
||||||
|
|||||||
Reference in New Issue
Block a user