From bc741734477f00b27196a8bd56c212d0fafc04d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 26 May 2021 11:57:30 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E58=E5=B0=8F=E5=8C=BA=E6=80=BB?= =?UTF-8?q?=E9=87=8F=E8=8E=B7=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/crawl/{ => 58}/City58.groovy | 0 .../shenjian/crawl/{ => 58}/Comm58.groovy | 0 .../shenjian/crawl/58/Count58Seaweed.groovy | 65 +++++++++ .../shenjian/crawl/{ => 58}/Seed58.groovy | 0 .../temp/shenjian/poi/SeaweedAroundPoi.groovy | 2 +- .../poi/SeaweedAroundPoiSingal.groovy | 128 ++++++++++++++++++ 6 files changed, 194 insertions(+), 1 deletion(-) rename projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/{ => 58}/City58.groovy (100%) rename projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/{ => 58}/Comm58.groovy (100%) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/Count58Seaweed.groovy rename projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/{ => 58}/Seed58.groovy (100%) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiSingal.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/City58.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/City58.groovy similarity index 100% rename from projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/City58.groovy rename to projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/City58.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Comm58.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/Comm58.groovy similarity index 100% rename from projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Comm58.groovy rename to projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/Comm58.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/Count58Seaweed.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/Count58Seaweed.groovy new file mode 100644 index 000000000..ca8e4379c --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/Count58Seaweed.groovy @@ -0,0 +1,65 @@ +package com.fangjia.script.temp.shenjian.crawl + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.crawl.InvokeHttpRequest +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.jsoup.Jsoup +import org.jsoup.nodes.Document + +/** + * 统计58小区总量 + * + * @author shenjian + * @since 2021/5/26 + */ +class Count58Seaweed { + + static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018) + + static void main(String[] args) { + crawlSeaweedCount() + println(count()) + } + + static crawlSeaweedCount() { + cityConn.find(new BasicDBObject("root", "58.com").append("district_count", new BasicDBObject('$eq', null)), + new BasicDBObject("_id", 1).append("city_short", 1).append("city", 1)) + .batchSize(100).each { obj -> + def id = obj._id + def city_short = obj.city_short + final def URL = "https://" + city_short + ".58.com/xiaoqu/" + try { + String res = ABuYunUtil.getReponseContent2(URL, "utf-8") + if (res) { + Document doc = Jsoup.parse(res); + String str = JQueryUtils.selectText(".total-info", doc) + // 117.60.236.41 + if (str) { + str = str.replace("共找到", "").replace("个小区", "").trim() + int count = Integer.parseInt(str) + cityConn.update(new BasicDBObject("_id", id), new BasicDBObject('$set', new BasicDBObject("district_count", count))) + println("当前处理城市" + obj.city) + } + } + } catch(Exception ex) { + ex.printStackTrace() + } + } + } + + static count() { + int number = 0; + cityConn.find(new BasicDBObject("root", "58.com"), new BasicDBObject("district_count", 1)) + .batchSize(100).each { obj -> + if (obj.district_count) { + number += obj.district_count + } + } + return number + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Seed58.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/Seed58.groovy similarity index 100% rename from projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Seed58.groovy rename to projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/Seed58.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy index 28ac4a0ab..016d7be75 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy @@ -29,7 +29,7 @@ class SeaweedAroundPoi { static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017) static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi" - static ExecutorService executorService = Executors.newFixedThreadPool(30) + static ExecutorService executorService = Executors.newFixedThreadPool(10) static void main(String[] args) { def types = listAllType() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiSingal.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiSingal.groovy new file mode 100644 index 000000000..be70b79e5 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiSingal.groovy @@ -0,0 +1,128 @@ +package com.fangjia.script.temp.shenjian.poi + + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.temp.shenjian.util.HttpUtils +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.wenyan.auto.KeywordSearchUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.collections.CollectionUtils +import org.bson.types.ObjectId + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 小区周边POI数据获取单线程方式 + * + * @author shenjian + * @since 2021/5/24 + */ +class SeaweedAroundPoiSingal { + + static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017) + static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi" + + static void main(String[] args) { + def types = listAllType() + //def types = "0601" + DBObject query = new BasicDBObject() + query.put("status", 1) + conn136.find(query).batchSize(30).each {objTemp -> + final def obj = objTemp + def city = obj.city + def lng = obj.lng + def lat = obj.lat + List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") + if (CollectionUtils.isNotEmpty(pois)) { + String jsonStr = JSONObject.toJSONString(pois) + HttpUtils.postForJson(URL, jsonStr) + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI"))) + } else { + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空"))) + } + println("已处理小区:" + obj) + } + } + + static getNewSeaweed() { + DBObject query = new BasicDBObject() + query.put("cat", "district") + query.put("visible", 0) + //query.put("_id", new ObjectId("50c96f82e4b0e15299cb018a")) + + DBObject filter = new BasicDBObject() + filter.put("_id", 1) + filter.put("city", 1) + filter.put("region", 1) + filter.put("name", 1) + filter.put("lng", 1) + filter.put("lat", 1) + + List list = [] + DbUtils.seaweed.find(query, filter).batchSize(10000).each {obj -> + if (conn136.count(new BasicDBObject("_id", obj._id)) > 0) { + // 已存在小区,无需同步 + } else { + DBObject newObj = new BasicDBObject() + newObj.put("_id", obj._id) + newObj.put("city", obj.city) + newObj.put("region", obj.region) + newObj.put("name", obj.name) + try { + newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng)) + newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat)) + } catch (Exception ex) { + print("经纬度为空:" + obj._id + " " + obj) + ex.printStackTrace() + } + newObj.put("status", 1) + newObj.put("flag", "未获取周边POI") + + list << newObj + if (list.size() == 5000) { + conn136.insert(list) + list.clear() + } + } + } + conn136.insert(list) + list.clear() + } + + static def getByIdAndInsert(String id) { + BasicDBObject obj = DbUtils.seaweed.findById(id); + DBObject newObj = new BasicDBObject() + newObj.put("_id", obj._id) + newObj.put("city", obj.city) + newObj.put("region", obj.region) + newObj.put("name", obj.name) + try { + newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng)) + newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat)) + } catch (Exception ex) { + print("经纬度为空:" + obj._id + " " + obj) + ex.printStackTrace() + } + newObj.put("status", 1) + newObj.put("flag", "未获取周边POI") + + conn136.insert(newObj) + } + + static def listAllType() { + StringBuilder ret = new StringBuilder() + for (def val : KeywordSearchUtils.KEYWORDS_MAP.values()) { + ret.append(val).append("|") + } + String result = ret.toString() + return result.substring(0, result.length() - 1) + } +}