diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy index e95cbadff..10e63221e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy @@ -2,6 +2,11 @@ package com.fangjia.script.temp.shenjian import com.alibaba.fastjson.JSONArray import com.alibaba.fastjson.JSONObject +import com.baidubce.http.ApiExplorerClient +import com.baidubce.http.AppSigner +import com.baidubce.http.HttpMethodName +import com.baidubce.model.ApiExplorerRequest +import com.baidubce.model.ApiExplorerResponse import com.fangjia.script.statistics.poi.AmapPoi import com.fangjia.script.temp.shenjian.util.SeaweedUtils import org.elasticsearch.geometry.utils.Geohash @@ -12,11 +17,28 @@ import org.elasticsearch.geometry.utils.Geohash class Temp { static void main(String[] args) { - String str = SeaweedUtils.encodeByGeoHash(106.569161, 29.498502) - println(str) - str = Geohash.stringEncode(106.569161, 29.498502, 10) - println(str) - // wm5zc4131rmf + String path = "https://goojia.api.bdymkt.com/estimate"; + ApiExplorerRequest request = new ApiExplorerRequest(HttpMethodName.GET, path); + request.setCredentials("fccf809a0f7748449154af926897aecb", "af1f0088a05c493ab20133ba470f87ef "); + + // 设置header参数 + request.addHeaderParameter("Content-Type", "application/json; charset=utf-8"); + + // 设置query参数 + request.addQueryParameter("address", "上海宝山呼玛三村"); + request.addQueryParameter("city", "上海"); + request.addQueryParameter("size", "100"); + + + ApiExplorerClient client = new ApiExplorerClient(new AppSigner()); + + try { + ApiExplorerResponse response = client.sendRequest(request); + // 返回结果格式为Json字符串 + System.out.println(response.getResult()); + } catch (Exception e) { + e.printStackTrace(); + } } // 万友七季城C区 经度纬度不同106.569161 29.498502 原始106.56881 29.50058 距离231.7003米 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/District58DetailCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/District58DetailCrawl.groovy new file mode 100644 index 000000000..d02c980c1 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/District58DetailCrawl.groovy @@ -0,0 +1,73 @@ +package com.fangjia.script.temp.shenjian.crawl + +import com.alibaba.fastjson.JSON +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.temp.shenjian.util.MessageUtils +import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import groovy.util.logging.Slf4j +import org.springframework.util.StopWatch + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 58小区详情爬取 + * + * @author shenjian* @since 2021/5/26 + */ +@Slf4j +class District58DetailCrawl { + + static DBCollection districtSeedConn = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018) + static ExecutorService executorService = Executors.newFixedThreadPool(10) + + static void main(String[] args) { + AtomicLong count = new AtomicLong() + CountDownLatch latch = new CountDownLatch(100000) + districtSeedConn.find(new BasicDBObject("root", "58.com").append("status", new BasicDBObject('$eq', null)), new BasicDBObject("district_id", 1).append("city_short", 1).append("_id", 1)) + .batchSize(1000).limit(100000).each { objTemp -> + def obj = objTemp + executorService.submit(new Runnable() { + @Override + void run() { + try { + final def URL = "https://" + obj.city_short + ".58.com/xiaoqu/" + obj.district_id + "/" + def res = ABuYunUtil.getReponseContent2(URL, "utf-8") + if (res && res.code == 200) { + String content = res.content + if (content.indexOf("访问过于频繁") == -1 && content.indexOf("请输入验证码") == -1) { + if (content.indexOf("您所访问的页面不存在") != -1) { + districtSeedConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "页面不存在"))) + } else { + JSON json = new JSONObject() + json.put("project", "district_detail_58") + json.put("url", URL) + + JSON result = new JSONObject() + result.put("content", (String) content) + json.put("result", result) + + byte[] packedMsg = MessageUtils.pack(json) + CommonUtils.sendMessage("result2parse2dev", packedMsg) + districtSeedConn.update(new BasicDBObject("_id", obj._id), + new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已抓取"))) + log.info("已抓取 {}", obj.district_id) + } + } + } + } catch (Exception ex) { + ex.printStackTrace() + } finally { + latch.countDown() + } + } + }) + } + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/DistrictDetailCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/DistrictDetailCrawl.groovy deleted file mode 100644 index 095456eed..000000000 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/DistrictDetailCrawl.groovy +++ /dev/null @@ -1,46 +0,0 @@ -package com.fangjia.script.temp.shenjian.crawl - -import com.alibaba.fastjson.JSON -import com.alibaba.fastjson.JSONObject -import com.fangjia.script.temp.shenjian.util.MessageUtils -import com.fangjia.script.util.ABuYunUtil -import com.fangjia.script.util.CommonUtils -import com.fangjia.script.util.MongoUtils -import com.mongodb.BasicDBObject -import com.mongodb.DBCollection - -/** - * 58小区详情爬取 - * - * @author shenjian - * @since 2021/5/26 - */ -class DistrictDetailCrawl { - - static DBCollection districtSeedConn = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018) - - static void main(String[] args) { - districtSeedConn.find(new BasicDBObject("root", "58.com").append("status", new BasicDBObject('$eq', null)), new BasicDBObject("district_id", 1).append("city_short", 1).append("_id", 1)) - .batchSize(1000).limit(1000).each { obj -> - final def URL = "https://" + obj.city_short + ".58.com/xiaoqu/" + obj.district_id + "/" - def res = ABuYunUtil.getReponseContent2(URL, "utf-8") - if (res && res.code == 200) { - String content = res.content - - JSON json = new JSONObject() - json.put("project", "district_detail_58") - json.put("url", URL) - - JSON result = new JSONObject() - result.put("content", (String)content) - json.put("result", result) - - byte[] packedMsg = MessageUtils.pack(json) - CommonUtils.sendMessage("result2parse", packedMsg) - districtSeedConn.update(new BasicDBObject("_id", obj._id), - new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已抓取"))) - println("已抓取" + obj.district_id) - } - } - } -} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy index 016d7be75..4d7a51343 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy @@ -29,11 +29,10 @@ class SeaweedAroundPoi { static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017) static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi" - static ExecutorService executorService = Executors.newFixedThreadPool(10) + static ExecutorService executorService = Executors.newFixedThreadPool(30) static void main(String[] args) { def types = listAllType() - //def types = "0601" DBObject query = new BasicDBObject() query.put("status", 1) conn136.find(query).batchSize(5000).each {objTemp -> @@ -46,6 +45,10 @@ class SeaweedAroundPoi { def lat = obj.lat List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") if (CollectionUtils.isNotEmpty(pois)) { + if (pois.get(0).id == "10044") { + executorService.shutdown() + throw new Error("当日调用已超限") + } String jsonStr = JSONObject.toJSONString(pois) HttpUtils.postForJson(URL, jsonStr) conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI"))) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index e384c35fb..3f234cc53 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -8,6 +8,8 @@ import com.fangjia.script.util.GeoHelper import com.fangjia.script.util.HttpRequestUtil import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import java.util.concurrent.ExecutorService + /** * 高德工具类 * @@ -138,7 +140,11 @@ class GaoDeUtils { } else if (json && json.infocode && json.infocode == "10044") { //throw new RuntimeException("开发者的日访问量超限,被系统自动封停,第二天0:00会自动解封") // 多线程情况下还是exit有效果,抛异常不会停止程序 - System.exit(1) + //System.exit(1) + AmapPoi amapPoi = new AmapPoi() + amapPoi.id = "10044" + result.add(amapPoi) + break; } else if (json && json.infocode && json.infocode == "10004") { // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 Thread.sleep(60000)