diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/FillLngLat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/FillLngLat.groovy new file mode 100644 index 000000000..d114be591 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/FillLngLat.groovy @@ -0,0 +1,27 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.temp.shenjian.util.EsUtils +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.util.ElasticUtils + +/** + * 补充学校经纬度 + * @author shenjian + * @since 2022/9/8 + */ +class FillLngLat { + + static void main(String[] args) { + ElasticUtils.executeSqlQuery("select * from dws_poi where location is null", { data -> + println(data) + def lngLat = GaoDeUtils.parserByGaode(data.city, data.region + data.name + data.address) + if (lngLat != null && lngLat.size() == 2 ) { + JSONObject jsonObject = new JSONObject() + jsonObject.put("lon", Double.valueOf(lngLat['lng'])) + jsonObject.put("lat", Double.valueOf(lngLat['lat'])) + EsUtils.bulkUpdateFieldByIds("dws_poi", "location", jsonObject, Arrays.asList(data.source_id)) + } + }) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy index f3c36ebd8..147d08336 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy @@ -10,6 +10,7 @@ import com.fangjia.script.temp.shenjian.util.HttpUtils import com.fangjia.script.temp.shenjian.util.SeaweedUtils import com.fangjia.script.util.DbUtils import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBList import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject @@ -27,15 +28,15 @@ import java.util.concurrent.Executors @BatchOwner(Watcher.ShenJian) class SeaweedAroundPoi extends BaseBatch { - static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017) + static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.235", 27018) static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoi" - static ExecutorService executorService = Executors.newFixedThreadPool(30) + //static ExecutorService executorService = Executors.newFixedThreadPool(2) static void main(String[] args) { def types = listAllType() //dealSpecialSeaweedAround(types) dealPoi(new BasicDBObject(), types) - executorService.shutdown() + //executorService.shutdown() } /** @@ -64,33 +65,40 @@ class SeaweedAroundPoi extends BaseBatch { } static dealPoi(DBObject query, def types) { - query.put("status", 1) - conn136.find(query).batchSize(5000).each {objTemp -> + query.put("canyin_status", 0) + conn136.find(query).batchSize(5000).limit(20000).noCursorTimeout(true).each {objTemp -> final def obj = objTemp - executorService.submit(new Runnable() { - @Override - void run() { +// executorService.submit(new Runnable() { +// @Override +// void run() { def city = obj.city - def lng = obj.lng - def lat = obj.lat - List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") - if (CollectionUtils.isNotEmpty(pois)) { - String jsonStr = JSONObject.toJSONString(pois) - HttpUtils.postForJson(URL, jsonStr) - conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI"))) - } else { - conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空"))) + if (obj.lng) { + //def lng = SeaweedUtils.formatLatLng(obj.lng) + //def lat = SeaweedUtils.formatLatLng(obj.lat) + // {'lon': 121.810877, 'lat': 31.737642} + def lng = 121.810877 + def lat = 31.737642 + List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") + if (CollectionUtils.isNotEmpty(pois)) { + String jsonStr = JSONObject.toJSONString(pois) + HttpUtils.postForJson(URL, jsonStr) + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("canyin_status", 0).append("flag", "已获取周边POI"))) + } else { + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("canyin_status", 2).append("flag", "周边POI为空"))) + } + println("已处理小区:" + obj) } - println("已处理小区:" + obj) } - }) - } +// }) +// } } static getNewSeaweed() { DBObject query = new BasicDBObject() query.put("cat", "district") query.put("visible", 0) + //query.put("city", new BasicDBObject('$in', ["连云港", "南通", "苏州", "无锡", "盐城", "扬州"])) + //query.put("city", "连云港") //query.put("_id", new ObjectId("50c96f82e4b0e15299cb018a")) DBObject filter = new BasicDBObject() @@ -163,7 +171,7 @@ class SeaweedAroundPoi extends BaseBatch { public static final Map KEYWORDS_MAP = new HashMap() { { - put("餐饮", "05"); +// put("餐饮", "05"); // put("地铁", "150500"); // put("公交", "1507"); // put("医院", "090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211"); @@ -172,8 +180,9 @@ class SeaweedAroundPoi extends BaseBatch { // put("银行", "1601|1603"); // put("大型超市", "0604"); // put("商场", "0601"); -// put("餐饮", "0501|0502|0503|0504"); + put("餐饮", "0501|0502|0503|0504"); // put("旅游景点", "110000"); +// put("公园广场", "110100|110101|110102|110103|110104|110105") // put("图书馆", "140500"); // put("政府机关", "130100|130101|130102|130103|130104|130105|130106|130107"); // put("丧葬设施", "071900|071901|071902|071903") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index f08b96c89..0aa5d9ce5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -146,11 +146,13 @@ class GaoDeUtils { while (result.size() < count) { params.page = "${page}".toString() page++ - params.key = this.getGaodeKey("v5/place/around") - JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v5/place/around", params); + params.key = this.getGaodeKey("v3/place/around") + params.offset = "100" + JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); if (json == null) { - params.key = this.getGaodeKey("v5/place/around") - json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v5/place/around", params); + params.key = this.getGaodeKey("v3/place/around") + params.offset = "100" + json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); } if (json && json.info && json.info == "OK") { count = json.getString("count").toInteger() @@ -258,7 +260,7 @@ class GaoDeUtils { } if (json && json.info && json.info == "OK") { try { - List ret = JSONObject.parseArray(json.pois.toString(), AmapPoi.class) + List ret = JSONObject.parseArray(json.pois.toString(), AmapPoiDetail.class) return ret } catch(Exception ex) { ex.printStackTrace() diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoiDetail.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoiDetail.java index 8192b359c..14922257d 100644 --- a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoiDetail.java +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoiDetail.java @@ -50,6 +50,21 @@ public class AmapPoiDetail { private BizExtDTO bizExt; @com.alibaba.fastjson.annotation.JSONField(name = "photos") private List photos; + @com.alibaba.fastjson.annotation.JSONField(name = "business") + private BusinessDTO business; + + @lombok.NoArgsConstructor + @lombok.Data + public static class BusinessDTO { + @com.alibaba.fastjson.annotation.JSONField(name = "business_area") + private String businessArea; + @com.alibaba.fastjson.annotation.JSONField(name = "keytag") + private String keytag; + @com.alibaba.fastjson.annotation.JSONField(name = "opentime_today") + private String opentimeToday; + @com.alibaba.fastjson.annotation.JSONField(name = "opentime_week") + private String opentimeWeek; + } @lombok.NoArgsConstructor @lombok.Data @@ -64,7 +79,7 @@ public class AmapPoiDetail { @lombok.Data public static class PhotosDTO { @com.alibaba.fastjson.annotation.JSONField(name = "title") - private List title; + private String title; @com.alibaba.fastjson.annotation.JSONField(name = "url") private String url; }