diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AllExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AllExport.groovy index 99bf24d62..0ed85bb0a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AllExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AllExport.groovy @@ -27,7 +27,7 @@ class AllExport { //3km范围内地铁站、公交站、银行、医院、商场、学校等名称及距离信息 static def types = "150500|1507|090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211|1601|0601|141200|141201|141202|141203|141204" - static def radius = "3000" + static def radius = 3000 static ExecutorService executorService = Executors.newFixedThreadPool(30) static void main(String[] args) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExport.groovy index fd3450f87..28cec6534 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExport.groovy @@ -27,7 +27,7 @@ class AroundExport { //3km范围内地铁站、公交站、银行、医院、商场、学校等名称及距离信息 static def types = "150500|1507|090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211|1601|0601|141200|141201|141202|141203|141204" - static def radius = "3000" + static def radius = 3000 //static ExecutorService executorService = Executors.newFixedThreadPool(30) // 高德地图中不存在周边的数据 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SavePoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SavePoi.groovy index 9da03d468..40c42f36a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SavePoi.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SavePoi.groovy @@ -21,7 +21,7 @@ class SavePoi { static DBCollection conn43 = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017) //3km范围内地铁站、公交站、银行、医院、商场、学校等名称及距离信息 static def types = "150500|1507|090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211|1601|0601|141200|141201|141202|141203|141204" - static def radius = "3000" + static def radius = 3000 static ExecutorService executorService = Executors.newFixedThreadPool(30) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy new file mode 100644 index 000000000..9b2599516 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy @@ -0,0 +1,124 @@ +package com.fangjia.script.temp.shenjian.poi + + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.temp.shenjian.util.HttpUtils +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.wenyan.auto.KeywordSearchUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.collections.CollectionUtils +import org.bson.types.ObjectId + +/** + * 小区周边POI数据获取 + * + * @author shenjian + * @since 2021/5/24 + */ +class SeaweedAroundPoi { + + static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017) + static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi" + + static void main(String[] args) { + def types = listAllType() + //def types = "0601" + DBObject query = new BasicDBObject() + query.put("status", 1) + conn136.find(query).batchSize(5000).each {obj -> + def city = obj.city + def lng = obj.lng + def lat = obj.lat + List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") + if (CollectionUtils.isNotEmpty(pois)) { + String jsonStr = JSONObject.toJSONString(pois) + HttpUtils.postForJson(URL, jsonStr) + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI"))) + } else { + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空"))) + } + println("已处理小区:" + obj) + } + } + + static getNewSeaweed() { + DBObject query = new BasicDBObject() + query.put("cat", "district") + query.put("visible", 0) + //query.put("_id", new ObjectId("50c96f82e4b0e15299cb018a")) + + DBObject filter = new BasicDBObject() + filter.put("_id", 1) + filter.put("city", 1) + filter.put("region", 1) + filter.put("name", 1) + filter.put("lng", 1) + filter.put("lat", 1) + + List list = [] + DbUtils.seaweed.find(query, filter).batchSize(10000).each {obj -> + if (conn136.count(new BasicDBObject("_id", obj._id)) > 0) { + // 已存在小区,无需同步 + } else { + DBObject newObj = new BasicDBObject() + newObj.put("_id", obj._id) + newObj.put("city", obj.city) + newObj.put("region", obj.region) + newObj.put("name", obj.name) + try { + newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng)) + newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat)) + } catch (Exception ex) { + print("经纬度为空:" + obj._id + " " + obj) + ex.printStackTrace() + } + newObj.put("status", 1) + newObj.put("flag", "未获取周边POI") + + list << newObj + if (list.size() == 5000) { + conn136.insert(list) + list.clear() + } + } + } + conn136.insert(list) + list.clear() + } + + static def getByIdAndInsert(String id) { + BasicDBObject obj = DbUtils.seaweed.findById(id); + DBObject newObj = new BasicDBObject() + newObj.put("_id", obj._id) + newObj.put("city", obj.city) + newObj.put("region", obj.region) + newObj.put("name", obj.name) + try { + newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng)) + newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat)) + } catch (Exception ex) { + print("经纬度为空:" + obj._id + " " + obj) + ex.printStackTrace() + } + newObj.put("status", 1) + newObj.put("flag", "未获取周边POI") + + conn136.insert(newObj) + } + + static def listAllType() { + StringBuilder ret = new StringBuilder() + for (def val : KeywordSearchUtils.KEYWORDS_MAP.values()) { + ret.append(val).append("|") + } + String result = ret.toString() + return result.substring(0, result.length() - 1) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/poi.json b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/poi.json new file mode 100644 index 000000000..4777a3761 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/poi.json @@ -0,0 +1,38 @@ +{ + "id": "B0FFF9RJ2O", + "parent": [], + "childtype": [], + "name": "丁兰广场", + "type": "购物服务;商场;购物中心", + "typecode": "060101", + "biz_type": [], + "address": "惠兰雅路与明珠街交汇处东北角", + "location": "120.231368,30.353127", + "tel": "0571-88111223", + "pname": "浙江省", + "cityname": "杭州市", + "adname": "上城区", + "importance": [], + "shopid": [], + "shopinfo": "0", + "poiweight": [], + "distance": "37", + "biz_ext": { + "rating": [], + "cost": "13953.00" + }, + "photos": [ + { + "title": [], + "url": "http://store.is.autonavi.com/showpic/f5dcc118938358031b45dea7e04faf45" + }, + { + "title": [], + "url": "http://store.is.autonavi.com/showpic/9a0fd6ceec63e030eac3ee14055206b5" + }, + { + "title": [], + "url": "http://store.is.autonavi.com/showpic/d9d2a87de3f00294665a2caeb3ee226f" + } + ] +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index ce6475a95..f772c3ab8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -1,12 +1,12 @@ package com.fangjia.script.temp.shenjian.util +import com.alibaba.fastjson.JSONArray +import com.alibaba.fastjson.JSONObject import com.fangjia.core.util.DataUtil +import com.fangjia.script.statistics.poi.AmapPoi import com.fangjia.script.util.GeoHelper import com.fangjia.script.util.HttpRequestUtil import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils -import net.sf.json.JSONObject - -import java.util.concurrent.ConcurrentHashMap /** * 高德工具类 @@ -16,7 +16,8 @@ import java.util.concurrent.ConcurrentHashMap */ class GaoDeUtils { - def static GAO_DE_AK = "2a1fb75990f2b39f6221e50892dd3988" + // 新获取key + def static GAO_DE_AK = "4e9592208336433ee1b7aa4d1d434418" def static codeNameMap = new HashMap(){{ put("150500", "地铁"); @@ -54,7 +55,7 @@ class GaoDeUtils { * @param types "150500|1507|090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211|1601|1603|0601"; @see com.fangjia.script.wenyan.auto.KeywordSearchUtils.KEYWORDS_MAP.get(kw) * @return */ - static List searchNearByPoi(String city, String types, Double lng, Double lat, String radius, String keywords){ + static List searchNearByPoi(String city, String types, Double lng, Double lat, Integer radius, String keywords){ def params = [:] params.key = GAO_DE_AK @@ -64,7 +65,7 @@ class GaoDeUtils { params.keywords = keywords } params.city = city - params.radius = radius + params.radius = String.valueOf(radius) params.location = "${lng},${lat}".toString() List result = [] @@ -80,6 +81,63 @@ class GaoDeUtils { if (json && json.info && json.info == "OK") { count = json.getString("count").toInteger() result.addAll(DataUtil.toMongoDbObj(json.pois)) + } else if (json && json.infocode && json.infocode == 10003) { + throw new RuntimeException("开发者的日访问量超限,被系统自动封停,第二天0:00会自动解封") + } else if (json && json.infocode && json.infocode == 10004) { + // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 + Thread.sleep(60000) + break; + } else { + break + } + } + return result + } + + /** + * 查找周边配套 + * @param city + * @param kw + * @param lng + * @param lat + * @param radius + * @param types + * @return + */ + static List searchNearDtoByPoi(String city, String types, Double lng, Double lat, Integer radius, String keywords){ + def params = [:] + params.key = GAO_DE_AK + + if (types) { + params.types = types + } else { + params.keywords = keywords + } + params.city = city + params.radius = String.valueOf(radius) + params.location = "${lng},${lat}".toString() + List result = [] + + int page = 1 + int count = 10000 + while (result.size() < count) { + params.page = "${page}".toString() + page++ + JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); + if (json == null) { + json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); + } + if (json && json.info && json.info == "OK") { + count = json.getString("count").toInteger() + JSONArray jsonArray = json.pois + List ret = JSONObject.parseArray(jsonArray.toJSONString(), AmapPoi.class); + result.addAll(ret) + } else if (json && json.infocode && json.infocode == 10003) { + throw new RuntimeException("开发者的日访问量超限,被系统自动封停,第二天0:00会自动解封") + } else if (json && json.infocode && json.infocode == 10004) { + // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 + Thread.sleep(60000) + break; } else { break } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy new file mode 100644 index 000000000..ff11d6991 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy @@ -0,0 +1,29 @@ +package com.fangjia.script.temp.shenjian.util + +import org.apache.commons.httpclient.HttpClient +import org.apache.commons.httpclient.methods.PostMethod +import org.apache.commons.httpclient.methods.StringRequestEntity + +/** + * + * @author shenjian + * @since 2021/5/25 + */ +class HttpUtils { + + static boolean postForJson(String url, String jsonString) { + HttpClient httpClient = new HttpClient() + StringRequestEntity requestEntity = new StringRequestEntity( + jsonString, + "application/json", + "UTF-8"); + + PostMethod postMethod = new PostMethod(url) + postMethod.setRequestEntity(requestEntity); + int statusCode = httpClient.executeMethod(postMethod); + if (statusCode == 200) { + return true + } + return false + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy index 2cb9a61b2..22673723b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy @@ -176,6 +176,14 @@ class SeaweedUtils { return "" } + /** + * 格式化经纬度 + */ + static def formatLatLng(def latOrLng) { + // 107.109878,29.192381 + return com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, latOrLng / 10000000000d) + } + static void main(String[] args) { def alias = ["金杏一村","金杏一村","金杏一村","金杏一村","金杏一村", "金杏二村"] println(formatAlias(alias)) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/auto/KeywordSearchUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/auto/KeywordSearchUtils.groovy index a2026322b..c3a0c4acf 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/auto/KeywordSearchUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/auto/KeywordSearchUtils.groovy @@ -204,6 +204,7 @@ public class KeywordSearchUtils{ put("旅游景点","110000"); put("体育场馆","0800|0801"); put("图书馆","140500"); + put("住宅区","120300|120301|120302|120303|120304"); }}; diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoi.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoi.java new file mode 100644 index 000000000..4a731fe11 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoi.java @@ -0,0 +1,71 @@ +package com.fangjia.script.statistics.poi; + +import java.util.List; + +/** + * @author shenjian + * @since 2021/5/25 + */ +@lombok.NoArgsConstructor +@lombok.Data +public class AmapPoi { + + @com.alibaba.fastjson.annotation.JSONField(name = "id") + private String id; + @com.alibaba.fastjson.annotation.JSONField(name = "parent") + private List parent; + @com.alibaba.fastjson.annotation.JSONField(name = "childtype") + private List childtype; + @com.alibaba.fastjson.annotation.JSONField(name = "name") + private String name; + @com.alibaba.fastjson.annotation.JSONField(name = "type") + private String type; + @com.alibaba.fastjson.annotation.JSONField(name = "typecode") + private String typecode; + @com.alibaba.fastjson.annotation.JSONField(name = "biz_type") + private List bizType; + @com.alibaba.fastjson.annotation.JSONField(name = "address") + private String address; + @com.alibaba.fastjson.annotation.JSONField(name = "location") + private String location; + @com.alibaba.fastjson.annotation.JSONField(name = "tel") + private String tel; + @com.alibaba.fastjson.annotation.JSONField(name = "pname") + private String pname; + @com.alibaba.fastjson.annotation.JSONField(name = "cityname") + private String cityname; + @com.alibaba.fastjson.annotation.JSONField(name = "adname") + private String adname; + @com.alibaba.fastjson.annotation.JSONField(name = "importance") + private List importance; + @com.alibaba.fastjson.annotation.JSONField(name = "shopid") + private List shopid; + @com.alibaba.fastjson.annotation.JSONField(name = "shopinfo") + private String shopinfo; + @com.alibaba.fastjson.annotation.JSONField(name = "poiweight") + private List poiweight; + @com.alibaba.fastjson.annotation.JSONField(name = "distance") + private String distance; + @com.alibaba.fastjson.annotation.JSONField(name = "biz_ext") + private BizExtDTO bizExt; + @com.alibaba.fastjson.annotation.JSONField(name = "photos") + private List photos; + + @lombok.NoArgsConstructor + @lombok.Data + public static class BizExtDTO { + @com.alibaba.fastjson.annotation.JSONField(name = "rating") + private List rating; + @com.alibaba.fastjson.annotation.JSONField(name = "cost") + private String cost; + } + + @lombok.NoArgsConstructor + @lombok.Data + public static class PhotosDTO { + @com.alibaba.fastjson.annotation.JSONField(name = "title") + private List title; + @com.alibaba.fastjson.annotation.JSONField(name = "url") + private String url; + } +}