diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedPoiDetail.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedPoiDetail.groovy new file mode 100644 index 000000000..feffbc12b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedPoiDetail.groovy @@ -0,0 +1,157 @@ +package com.fangjia.script.temp.shenjian.poi + + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.statistics.poi.AmapPoiDetail +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.temp.shenjian.util.HttpUtils +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.wenyan.auto.KeywordSearchUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import com.mongodb.client.model.Filters +import com.mongodb.client.model.UpdateOneModel +import com.mongodb.client.model.UpdateOptions +import com.mongodb.client.model.WriteModel +import org.apache.commons.collections.CollectionUtils +import org.apache.commons.lang.StringUtils +import org.bson.Document +import org.bson.conversions.Bson +import org.bson.types.ObjectId + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 小区POI信息精确查找 + * + * @author shenjian + * @since 2022/5/25 + */ +class SeaweedPoiDetail { + + static def conn136 = MongoUtils.getCollectionByName3("fangjia", "seaweed_poi_sync", "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017") + static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoiDetail" + // 住宅类 + static final String typecodes = "120000|120100|120200|120201|120202|120203|120300|120301|120302|120303|120304"; + static final ExecutorService executorService = Executors.newFixedThreadPool(20) + + static void main(String[] args) { + DBObject query = new BasicDBObject() + query.put("status", 1) + query.put("city", "东莞") + conn136.find(query).batchSize(1000).each {objTemp -> + final def obj = objTemp + executorService.submit(new Runnable() { + @Override + void run() { + def city = obj.city + def region = obj.region + def name = obj.name + def alias = obj.alias + def address = obj.address + def addr_alias = obj.addr_alias + StringJoiner stringJoiner = new StringJoiner("|") + // 主小区名搜索 + stringJoiner.add(name) + // 根据别名搜 + if (alias != null && CollectionUtils.isNotEmpty(alias)) { + for (String str : alias) { + stringJoiner.add(str) + } + } + // 根据路名搜 + if (StringUtils.isNotBlank(address)) { + stringJoiner.add(address) + } + // 根据路别名搜 + if (addr_alias != null && CollectionUtils.isNotEmpty(addr_alias)) { + for (String str : addr_alias) { + stringJoiner.add(str) + } + } + + String keywords = stringJoiner.toString() + List pois = GaoDeUtils.searchPoi(city, keywords, region, typecodes) + + List detailList = new ArrayList<>() + for (AmapPoi amapPoi : pois) { + List amapPoiList = GaoDeUtils.getAmapDetailById(amapPoi.getId()); + if (CollectionUtils.isNotEmpty(amapPoiList)) { + detailList.addAll(amapPoiList) + } + } + if (CollectionUtils.isNotEmpty(detailList)) { + String jsonStr = JSONObject.toJSONString(detailList) + HttpUtils.postForJson(URL, jsonStr) + conn136.updateOne(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取到"))) + } else { + conn136.updateOne(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "未获取到小区"))) + } + println("已处理小区:" + obj) + } + }) + } + executorService.shutdown() + } + + static getNewSeaweed() { + DBObject query = new BasicDBObject() + query.put("cat", "district") + query.put("visible", 0) + + DBObject filter = new BasicDBObject() + filter.put("_id", 1) + filter.put("city", 1) + filter.put("region", 1) + filter.put("name", 1) + filter.put("alias", 1) + filter.put("address", 1) + filter.put("addr_alias", 1) + filter.put("lng", 1) + filter.put("lat", 1) + + List> writeModelList = new ArrayList<>() + def count = 0 + DbUtils.seaweed.find(query, filter).batchSize(10000).each {obj -> + Document updateBson = new Document() + Document update = new Document() + update.put("city", obj.city) + update.put("region", obj.region) + update.put("name", obj.name) + update.put("alias", obj.alias) + update.put("address", obj.address) + update.put("addr_alias", obj.addr_alias) + try { + update.put("lng", SeaweedUtils.formatLatLng(obj.lng)) + update.put("lat", SeaweedUtils.formatLatLng(obj.lat)) + } catch (Exception ex) { + print("经纬度为空:" + obj._id + " " + obj) + } + update.put("status", 1) + update.put("flag", "未精确获取高德信息") + updateBson.append('$set', update) + + Bson filterBson = Filters.eq("_id", obj._id) + UpdateOptions updateOptions = new UpdateOptions(); + updateOptions.upsert(true) + UpdateOneModel updateOneModel = new UpdateOneModel(filterBson, updateBson, updateOptions); + writeModelList.add(updateOneModel) + if (writeModelList.size() == 5000) { + conn136.bulkWrite(writeModelList) + count += 5000 + println(count) + writeModelList.clear() + } + } + if (writeModelList.size() > 0) { + conn136.bulkWrite(writeModelList) + writeModelList.clear() + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 49c7c18de..aae3318c9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -7,10 +7,12 @@ import com.fangjia.core.util.DataUtil import com.fangjia.core.util.HttpRequest import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.statistics.poi.AmapPoiDetail import com.fangjia.script.util.GeoHelper import com.fangjia.script.util.HttpRequestUtil import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import org.apache.commons.lang.StringUtils +import org.openqa.selenium.json.Json import java.util.concurrent.atomic.AtomicInteger @@ -232,6 +234,35 @@ class GaoDeUtils { return Collections.emptyList() } + /** + * 获取详情 + */ + static List getAmapDetailById(id){ + def params = [:] + params.key = GAO_DE_AK[keyUsed.intValue()] + params.id = id + params.output = 'json' + JSONObject json = HttpRequestUtil.getJsonContent("https://restapi.amap.com/v3/place/detail", params); + if (json == null) { + json = HttpRequestUtil.getJsonContent("https://restapi.amap.com/v3/place/detail", params); + } + if (json && json.info && json.info == "OK") { + try { + List ret = JSONObject.parseArray(json.pois.toString(), AmapPoi.class) + return ret + } catch(Exception ex) { + ex.printStackTrace() + } + } else if (json && json.infocode && json.infocode == "10004") { + // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 + Thread.sleep(60000) + } else if (json && json.infocode && json.infocode == "10017") { + // 所请求的资源不可用 + System.exit(1) + } + return Collections.emptyList() + } + /** * 查找关键字经纬度 */ diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoiDetail.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoiDetail.java new file mode 100644 index 000000000..8192b359c --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoiDetail.java @@ -0,0 +1,71 @@ +package com.fangjia.script.statistics.poi; + +import java.util.List; + +/** + * @author shenjian + * @since 2022/5/25 + */ +@lombok.NoArgsConstructor +@lombok.Data +public class AmapPoiDetail { + + @com.alibaba.fastjson.annotation.JSONField(name = "id") + private String id; + @com.alibaba.fastjson.annotation.JSONField(name = "parent") + private List parent; + @com.alibaba.fastjson.annotation.JSONField(name = "childtype") + private List childtype; + @com.alibaba.fastjson.annotation.JSONField(name = "name") + private String name; + @com.alibaba.fastjson.annotation.JSONField(name = "type") + private String type; + @com.alibaba.fastjson.annotation.JSONField(name = "typecode") + private String typecode; + @com.alibaba.fastjson.annotation.JSONField(name = "biz_type") + private List bizType; + @com.alibaba.fastjson.annotation.JSONField(name = "address") + private String address; + @com.alibaba.fastjson.annotation.JSONField(name = "location") + private String location; + @com.alibaba.fastjson.annotation.JSONField(name = "tel") + private String tel; + @com.alibaba.fastjson.annotation.JSONField(name = "pname") + private String pname; + @com.alibaba.fastjson.annotation.JSONField(name = "cityname") + private String cityname; + @com.alibaba.fastjson.annotation.JSONField(name = "adname") + private String adname; + @com.alibaba.fastjson.annotation.JSONField(name = "importance") + private List importance; + @com.alibaba.fastjson.annotation.JSONField(name = "shopid") + private List shopid; + @com.alibaba.fastjson.annotation.JSONField(name = "shopinfo") + private String shopinfo; + @com.alibaba.fastjson.annotation.JSONField(name = "poiweight") + private List poiweight; + @com.alibaba.fastjson.annotation.JSONField(name = "distance") + private String distance; + @com.alibaba.fastjson.annotation.JSONField(name = "biz_ext") + private BizExtDTO bizExt; + @com.alibaba.fastjson.annotation.JSONField(name = "photos") + private List photos; + + @lombok.NoArgsConstructor + @lombok.Data + public static class BizExtDTO { + @com.alibaba.fastjson.annotation.JSONField(name = "rating") + private List rating; + @com.alibaba.fastjson.annotation.JSONField(name = "cost") + private String cost; + } + + @lombok.NoArgsConstructor + @lombok.Data + public static class PhotosDTO { + @com.alibaba.fastjson.annotation.JSONField(name = "title") + private List title; + @com.alibaba.fastjson.annotation.JSONField(name = "url") + private String url; + } +}