From 7042809ecd9bf1fa3af87232fa71c16f124fd914 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 18 Jun 2021 18:08:24 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=A4=9A=E7=BA=BF=E7=A8=8B?= =?UTF-8?q?=E5=A4=9Akey=E5=81=9C=E6=AD=A2bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- projects/fangjia-script/pom.xml | 7 +- .../temp/shenjian/poi/SeaweedAroundPoi.groovy | 185 ++++++++++++++++ .../temp/shenjian/util/GaoDeUtils.groovy | 186 ++++++++++++++++ .../temp/shenjian/util/HttpUtils.groovy | 29 +++ .../temp/shenjian/util/SeaweedUtils.groovy | 208 ++++++++++++++++++ .../script/statistics/poi/AmapPoi.java | 71 ++++++ 6 files changed, 685 insertions(+), 1 deletion(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoi.java diff --git a/projects/fangjia-script/pom.xml b/projects/fangjia-script/pom.xml index ee26c1830..75ca2b1ff 100644 --- a/projects/fangjia-script/pom.xml +++ b/projects/fangjia-script/pom.xml @@ -482,7 +482,12 @@ j2v8_win32_x86_64 4.6.0 - + + org.projectlombok + lombok + 1.18.20 + provided + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy new file mode 100644 index 000000000..00ee277ea --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy @@ -0,0 +1,185 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.temp.shenjian.util.HttpUtils +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.collections.CollectionUtils + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 小区周边POI数据获取 + * + * @author shenjian + * @since 2021/5/24 + */ +@BatchOwner(Watcher.ShenJian) +class SeaweedAroundPoi extends BaseBatch { + + static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017) + static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi" + static ExecutorService executorService = Executors.newFixedThreadPool(30) + + static void main(String[] args) { + def types = listAllType() + //dealSpecialSeaweedAround(types) + dealPoi(new BasicDBObject(), types) + executorService.shutdown() + } + + /** + * 有限处理招标城市 + */ + static dealSpecialSeaweedAround(def types) { + def cities = ["重庆", "达州", "曲靖", "昆明", "大理", "迪庆州", "福安", "平潭", "三明", "石狮", "柳州", "苏州"] + for (def city : cities) { + DBObject q = new BasicDBObject(); + q.put("city", city); + if (city == "福安") { + city = "宁德" + q.put("city", city) + q.put("region", "福安") + } else if (city == "平潭") { + city = "福州" + q.put("city", city) + q.put("region", "平潭") + } else if (city == "石狮") { + city = "泉州" + q.put("city", city) + q.put("region", "石狮") + } + dealPoi(q, types) + } + } + + static dealPoi(DBObject query, def types) { + query.put("status", 1) + conn136.find(query).batchSize(5000).each {objTemp -> + final def obj = objTemp + executorService.submit(new Runnable() { + @Override + void run() { + def city = obj.city + def lng = obj.lng + def lat = obj.lat + List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") + if (CollectionUtils.isNotEmpty(pois)) { + String jsonStr = JSONObject.toJSONString(pois) + HttpUtils.postForJson(URL, jsonStr) + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI"))) + } else { + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空"))) + } + println("已处理小区:" + obj) + } + }) + } + } + + static getNewSeaweed() { + DBObject query = new BasicDBObject() + query.put("cat", "district") + query.put("visible", 0) + //query.put("_id", new ObjectId("50c96f82e4b0e15299cb018a")) + + DBObject filter = new BasicDBObject() + filter.put("_id", 1) + filter.put("city", 1) + filter.put("region", 1) + filter.put("name", 1) + filter.put("lng", 1) + filter.put("lat", 1) + + List list = [] + DbUtils.seaweed.find(query, filter).batchSize(10000).each {obj -> + if (conn136.count(new BasicDBObject("_id", obj._id)) > 0) { + // 已存在小区,无需同步 + } else { + DBObject newObj = new BasicDBObject() + newObj.put("_id", obj._id) + newObj.put("city", obj.city) + newObj.put("region", obj.region) + newObj.put("name", obj.name) + try { + newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng)) + newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat)) + } catch (Exception ex) { + print("经纬度为空:" + obj._id + " " + obj) + ex.printStackTrace() + } + newObj.put("status", 1) + newObj.put("flag", "未获取周边POI") + + list << newObj + if (list.size() == 5000) { + conn136.insert(list) + list.clear() + } + } + } + conn136.insert(list) + list.clear() + } + + static def getByIdAndInsert(String id) { + BasicDBObject obj = DbUtils.seaweed.findById(id); + DBObject newObj = new BasicDBObject() + newObj.put("_id", obj._id) + newObj.put("city", obj.city) + newObj.put("region", obj.region) + newObj.put("name", obj.name) + try { + newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng)) + newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat)) + } catch (Exception ex) { + print("经纬度为空:" + obj._id + " " + obj) + ex.printStackTrace() + } + newObj.put("status", 1) + newObj.put("flag", "未获取周边POI") + + conn136.insert(newObj) + } + + static def listAllType() { + StringBuilder ret = new StringBuilder() + for (def val : KEYWORDS_MAP.values()) { + ret.append(val).append("|") + } + String result = ret.toString() + return result.substring(0, result.length() - 1) + } + + public static final Map KEYWORDS_MAP = new HashMap(){{ + put("地铁", "150500"); + put("公交", "1507"); + put("医院", "090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211"); + put("写字楼", "120201"); + put("学校","141200|141201|141202|141203|141204"); + put("银行","1601|1603"); + put("大型超市","0604"); + put("商场","0601"); + put("餐饮","0501|0502|0503|0504"); + put("旅游景点","110000"); + put("图书馆","140500"); + }}; + + @Override + void process(Object args) { + def types = listAllType() + //dealSpecialSeaweedAround(types) + dealPoi(new BasicDBObject(), types) + executorService.shutdown() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy new file mode 100644 index 000000000..74505e6b5 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -0,0 +1,186 @@ +package com.fangjia.script.temp.shenjian.util + +import com.alibaba.fastjson.JSONArray +import com.alibaba.fastjson.JSONObject +import com.fangjia.core.util.DataUtil +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.util.GeoHelper +import com.fangjia.script.util.HttpRequestUtil +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils + +import java.util.concurrent.atomic.AtomicInteger + +/** + * 高德工具类 + * + * @author shenjian + * @since 2021/5/15 + */ +class GaoDeUtils { + + // 新获取key + def static GAO_DE_AK = ["4e9592208336433ee1b7aa4d1d434418", "35925efd7333f1df52145e51b8e4e56a"] + def static AtomicInteger keyUsed = new AtomicInteger(0) + + def static codeNameMap = new HashMap(){{ + put("150500", "地铁"); + put("1507", "公交"); + put("090100", "医院") + put("090101", "医院") + put("090200", "医院") + put("090202", "医院") + put("090202", "医院") + put("090204", "医院") + put("090205", "医院") + put("090206", "医院") + put("090206", "医院") + put("090208", "医院") + put("090209", "医院") + put("090210", "医院") + put("090211", "医院") + put("141200", "学校") + put("141201", "学校") + put("141202", "学校") + put("141203", "学校") + put("141204", "学校") + put("1601", "银行") + put("1603", "银行") + put("0601", "商场") + }}; + + /** + * 查找周边配套 + * @param city + * @param kw + * @param lng + * @param lat + * @param radius + * @param types "150500|1507|090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211|1601|1603|0601"; @see com.fangjia.script.wenyan.auto.KeywordSearchUtils.KEYWORDS_MAP.get(kw) + * @return + */ + static List searchNearByPoi(String city, String types, Double lng, Double lat, Integer radius, String keywords){ + def params = [:] + params.key = GAO_DE_AK[keyUsed] + + if (types) { + params.types = types + } else { + params.keywords = keywords + } + params.city = city + params.radius = String.valueOf(radius) + params.location = "${lng},${lat}".toString() + List result = [] + + int page = 1 + int count = 10000 + while (result.size() < count) { + params.page = "${page}".toString() + page++ + JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); + if (json == null) { + json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); + } + if (json && json.info && json.info == "OK") { + count = json.getString("count").toInteger() + result.addAll(DataUtil.toMongoDbObj(json.pois)) + } else if (json && json.infocode && json.infocode == 10003) { + throw new RuntimeException("开发者的日访问量超限,被系统自动封停,第二天0:00会自动解封") + } else if (json && json.infocode && json.infocode == 10004) { + // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 + Thread.sleep(60000) + break; + } else { + break + } + } + return result + } + + /** + * 查找周边配套 + * @param city + * @param kw + * @param lng + * @param lat + * @param radius + * @param types + * @return + */ + static List searchNearDtoByPoi(String city, String types, Double lng, Double lat, Integer radius, String keywords){ + def params = [:] + params.key = GAO_DE_AK[keyUsed.intValue()] + + if (types) { + params.types = types + } else { + params.keywords = keywords + } + params.city = city + params.radius = String.valueOf(radius) + params.location = "${lng},${lat}".toString() + List result = [] + + int page = 1 + int count = 10000 + while (result.size() < count) { + params.page = "${page}".toString() + page++ + JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); + if (json == null) { + json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); + } + if (json && json.info && json.info == "OK") { + count = json.getString("count").toInteger() + try { + List ret = JSONObject.parseArray(json.pois.toString(), AmapPoi.class) + result.addAll(ret) + } catch(Exception ex) { + ex.printStackTrace() + } + } else if (json && json.infocode && json.infocode == "10044") { + synchronized (this) { + for (int i = 0; i < GAO_DE_AK.size(); i++) { + if (GAO_DE_AK[i] == params.key) { + if (i + 1 > GAO_DE_AK.size()) { + System.exit(1) + } + keyUsed = new AtomicInteger(i+1) + } + } + break; + } + } else if (json && json.infocode && json.infocode == "10004") { + // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 + Thread.sleep(60000) + break; + } else { + break + } + } + return result + } + + /** + * 获取经纬度之间距离 + */ + static def getDistance(double Lat, double Lng, double Lat1, double Lng1) { + return GeoHelper.GetDistance(Lat, Lng, Lat1, Lng1) + } + + /** + * 获取地址经纬度 + */ + static def parserByGaode(def city, def address) { + JSONObject json = GaoDeApiUtils.geo(city, address) + if (json && json.status == "1") { + def ret = json.geocodes + if (ret) { + def str = ret.get(0).location + def level = ret.get(0).level + def arrs = str.split(",") + return ["lng": arrs[0], "lat": arrs[1]] + } + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy new file mode 100644 index 000000000..ff11d6991 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy @@ -0,0 +1,29 @@ +package com.fangjia.script.temp.shenjian.util + +import org.apache.commons.httpclient.HttpClient +import org.apache.commons.httpclient.methods.PostMethod +import org.apache.commons.httpclient.methods.StringRequestEntity + +/** + * + * @author shenjian + * @since 2021/5/25 + */ +class HttpUtils { + + static boolean postForJson(String url, String jsonString) { + HttpClient httpClient = new HttpClient() + StringRequestEntity requestEntity = new StringRequestEntity( + jsonString, + "application/json", + "UTF-8"); + + PostMethod postMethod = new PostMethod(url) + postMethod.setRequestEntity(requestEntity); + int statusCode = httpClient.executeMethod(postMethod); + if (statusCode == 200) { + return true + } + return false + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy new file mode 100644 index 000000000..9fa4739f8 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy @@ -0,0 +1,208 @@ +package com.fangjia.script.temp.shenjian.util + +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.datastore.FieldValidatorUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.elasticsearch.geometry.utils.Geohash + +import java.util.concurrent.ConcurrentHashMap + +/** + * 小区工具类 + * + * @author shenjian + * @since 2021/5/17 + */ +class SeaweedUtils { + + static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017) + static def map = new ConcurrentHashMap() + + static getRegionByCode(def district_code) { + def region; + if (map.get("district_code") != null) { + region = map.get("district_code") + } else { + cityMapConn.find(new BasicDBObject("district_code", district_code)).each { obj -> + region = obj.fangjia_district + map.put("district_code", region) + } + } + return region + } + + /** + * 标准化输出绿化率 + */ + static formatVirescencePercent(def virescence_percent) { + // 绿化率 + if (virescence_percent && FieldValidatorUtils.checkValue(virescence_percent)) { + double val = Double.valueOf(virescence_percent) + if (val < 1) { + return val * 100 + } + if (val == 1) { + return val * 10 + } + if (val >= 70) { + return 65 + } + return val + } + return "" + } + + /** + * 标准化容积率输出 + */ + static formatCubagePercent(def cubage_percent) { + if (cubage_percent && FieldValidatorUtils.checkValue(cubage_percent)) { + double val = Double.valueOf(cubage_percent) + if (val >= 7 && val < 8) { + return 6.2 + } + if (val >= 8 && val < 9) { + return 6.3 + } + if (val >= 9 && val < 10) { + return 6.5 + } + if (val >= 10) { + return 6.7 + } + return val + } + return "" + } + + /** + * 标准化车位输出 + */ + static formatCarSiteCount(def car_site_count) { + if (car_site_count && FieldValidatorUtils.checkValue(car_site_count)) { + long val = Long.valueOf(car_site_count) + if (val > 2000 || val < 100) { + return "" + } + return val; + } + return "" + } + + /** + * 标准化建筑面积输出 + */ + static formatBuildArea(def build_area) { + if (build_area && FieldValidatorUtils.checkValue(build_area)) { + double val = build_area + if (val > 1000) { + return val + } + } + return "" + } + + /** + * 标准化总占地面积输出 + */ + static formatAllArea(def all_area) { + if (all_area) { + Double formattedVal = 0.0; + if (all_area.toString().indexOf("公顷") != -1) { + def val = all_area.toString() + formattedVal = (Double.valueOf(val.substring(0, val.indexOf("公顷"))) * 10000) + } else if(FieldValidatorUtils.checkValue(all_area)) { + formattedVal = Double.valueOf(all_area) + } + if (formattedVal > 100000 || formattedVal == 0.0) { + return "" + } + return formattedVal + } + } + + /** + * 标准化户数输出 + */ + static formatHouseholdCount(def household_count) { + if (household_count && FieldValidatorUtils.checkValue(household_count) ) { + if (Long.valueOf(household_count) < 50) { + return "" // 户数 + } + return household_count + } + } + + /** + * 计算车位比 + */ + static calcCarAndHouseholdRadio(def car_site_count, def household_count) { + if (SeaweedUtils.formatCarSiteCount(car_site_count) && SeaweedUtils.formatHouseholdCount(household_count)) { + long formattedCarSiteCount = SeaweedUtils.formatCarSiteCount(car_site_count) + long formattedHouseholdCount = SeaweedUtils.formatHouseholdCount(household_count) + double val = DataHandleUtils.getDoubleRound(1, (Double.valueOf(formattedCarSiteCount) / Double.valueOf(formattedHouseholdCount))) + if (val == 0.0) { + val = DataHandleUtils.getDoubleRound(2, (Double.valueOf(formattedCarSiteCount) / Double.valueOf(formattedHouseholdCount))) + } + if (val == 0.0) { + val = DataHandleUtils.getDoubleRound(3, (Double.valueOf(formattedCarSiteCount) / Double.valueOf(formattedHouseholdCount))) + } + if (val == 1.0) { + return "1:1" + } + return "1:" + val + } + return "" + } + + /** + * 标准化别名输出 + */ + static formatAlias(def alias) { + if (alias) { + // 小区可能存在多个别名 + def temp = [] + int len = 0 + for (def a : alias) { + len += a.toString().length() + if (len > 20) { + return temp.join(",") + } + temp.add(a) + } + return temp.join(",") + } + return "" + } + + /** + * 格式化经纬度 + */ + static def formatLatLng(def latOrLng) { + // 107.109878,29.192381 + return com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, latOrLng / 10000000000d) + } + + /** + * 小区经纬度geohash编码 + * @param args + */ + static def encodeByGeoHash(double lon, double lat, int level=7) { + return Geohash.stringEncode(lon, lat, level) + } + + /** + * 小区经纬度geohash解码 + * @param args + */ + static def decodeByGeoHash(String geohash) { + return Geohash.decodeLongitude(geohash) + "," + Geohash.decodeLatitude(geohash) + } + + static void main(String[] args) { + def alias = ["金杏一村","金杏一村","金杏一村","金杏一村","金杏一村", "金杏二村"] + println(formatAlias(alias)) + } +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoi.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoi.java new file mode 100644 index 000000000..4a731fe11 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoi.java @@ -0,0 +1,71 @@ +package com.fangjia.script.statistics.poi; + +import java.util.List; + +/** + * @author shenjian + * @since 2021/5/25 + */ +@lombok.NoArgsConstructor +@lombok.Data +public class AmapPoi { + + @com.alibaba.fastjson.annotation.JSONField(name = "id") + private String id; + @com.alibaba.fastjson.annotation.JSONField(name = "parent") + private List parent; + @com.alibaba.fastjson.annotation.JSONField(name = "childtype") + private List childtype; + @com.alibaba.fastjson.annotation.JSONField(name = "name") + private String name; + @com.alibaba.fastjson.annotation.JSONField(name = "type") + private String type; + @com.alibaba.fastjson.annotation.JSONField(name = "typecode") + private String typecode; + @com.alibaba.fastjson.annotation.JSONField(name = "biz_type") + private List bizType; + @com.alibaba.fastjson.annotation.JSONField(name = "address") + private String address; + @com.alibaba.fastjson.annotation.JSONField(name = "location") + private String location; + @com.alibaba.fastjson.annotation.JSONField(name = "tel") + private String tel; + @com.alibaba.fastjson.annotation.JSONField(name = "pname") + private String pname; + @com.alibaba.fastjson.annotation.JSONField(name = "cityname") + private String cityname; + @com.alibaba.fastjson.annotation.JSONField(name = "adname") + private String adname; + @com.alibaba.fastjson.annotation.JSONField(name = "importance") + private List importance; + @com.alibaba.fastjson.annotation.JSONField(name = "shopid") + private List shopid; + @com.alibaba.fastjson.annotation.JSONField(name = "shopinfo") + private String shopinfo; + @com.alibaba.fastjson.annotation.JSONField(name = "poiweight") + private List poiweight; + @com.alibaba.fastjson.annotation.JSONField(name = "distance") + private String distance; + @com.alibaba.fastjson.annotation.JSONField(name = "biz_ext") + private BizExtDTO bizExt; + @com.alibaba.fastjson.annotation.JSONField(name = "photos") + private List photos; + + @lombok.NoArgsConstructor + @lombok.Data + public static class BizExtDTO { + @com.alibaba.fastjson.annotation.JSONField(name = "rating") + private List rating; + @com.alibaba.fastjson.annotation.JSONField(name = "cost") + private String cost; + } + + @lombok.NoArgsConstructor + @lombok.Data + public static class PhotosDTO { + @com.alibaba.fastjson.annotation.JSONField(name = "title") + private List title; + @com.alibaba.fastjson.annotation.JSONField(name = "url") + private String url; + } +}