新增周边POI数据获取

This commit is contained in:
沈健
2021-05-25 11:22:13 +08:00
parent 95bcb3739f
commit a00200c12e
10 changed files with 338 additions and 9 deletions
@@ -27,7 +27,7 @@ class AllExport {
//3km范围内地铁站、公交站、银行、医院、商场、学校等名称及距离信息
static def types = "150500|1507|090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211|1601|0601|141200|141201|141202|141203|141204"
static def radius = "3000"
static def radius = 3000
static ExecutorService executorService = Executors.newFixedThreadPool(30)
static void main(String[] args) {
@@ -27,7 +27,7 @@ class AroundExport {
//3km范围内地铁站、公交站、银行、医院、商场、学校等名称及距离信息
static def types = "150500|1507|090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211|1601|0601|141200|141201|141202|141203|141204"
static def radius = "3000"
static def radius = 3000
//static ExecutorService executorService = Executors.newFixedThreadPool(30)
// 高德地图中不存在周边的数据
@@ -21,7 +21,7 @@ class SavePoi {
static DBCollection conn43 = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017)
//3km范围内地铁站、公交站、银行、医院、商场、学校等名称及距离信息
static def types = "150500|1507|090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211|1601|0601|141200|141201|141202|141203|141204"
static def radius = "3000"
static def radius = 3000
static ExecutorService executorService = Executors.newFixedThreadPool(30)
@@ -0,0 +1,124 @@
package com.fangjia.script.temp.shenjian.poi
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.statistics.poi.AmapPoi
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.temp.shenjian.util.HttpUtils
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.wenyan.auto.KeywordSearchUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.apache.commons.collections.CollectionUtils
import org.bson.types.ObjectId
/**
* 小区周边POI数据获取
*
* @author shenjian
* @since 2021/5/24
*/
class SeaweedAroundPoi {
static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017)
static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi"
static void main(String[] args) {
def types = listAllType()
//def types = "0601"
DBObject query = new BasicDBObject()
query.put("status", 1)
conn136.find(query).batchSize(5000).each {obj ->
def city = obj.city
def lng = obj.lng
def lat = obj.lat
List<AmapPoi> pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "")
if (CollectionUtils.isNotEmpty(pois)) {
String jsonStr = JSONObject.toJSONString(pois)
HttpUtils.postForJson(URL, jsonStr)
conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI")))
} else {
conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空")))
}
println("已处理小区:" + obj)
}
}
static getNewSeaweed() {
DBObject query = new BasicDBObject()
query.put("cat", "district")
query.put("visible", 0)
//query.put("_id", new ObjectId("50c96f82e4b0e15299cb018a"))
DBObject filter = new BasicDBObject()
filter.put("_id", 1)
filter.put("city", 1)
filter.put("region", 1)
filter.put("name", 1)
filter.put("lng", 1)
filter.put("lat", 1)
List list = []
DbUtils.seaweed.find(query, filter).batchSize(10000).each {obj ->
if (conn136.count(new BasicDBObject("_id", obj._id)) > 0) {
// 已存在小区,无需同步
} else {
DBObject newObj = new BasicDBObject()
newObj.put("_id", obj._id)
newObj.put("city", obj.city)
newObj.put("region", obj.region)
newObj.put("name", obj.name)
try {
newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng))
newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat))
} catch (Exception ex) {
print("经纬度为空:" + obj._id + " " + obj)
ex.printStackTrace()
}
newObj.put("status", 1)
newObj.put("flag", "未获取周边POI")
list << newObj
if (list.size() == 5000) {
conn136.insert(list)
list.clear()
}
}
}
conn136.insert(list)
list.clear()
}
static def getByIdAndInsert(String id) {
BasicDBObject obj = DbUtils.seaweed.findById(id);
DBObject newObj = new BasicDBObject()
newObj.put("_id", obj._id)
newObj.put("city", obj.city)
newObj.put("region", obj.region)
newObj.put("name", obj.name)
try {
newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng))
newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat))
} catch (Exception ex) {
print("经纬度为空:" + obj._id + " " + obj)
ex.printStackTrace()
}
newObj.put("status", 1)
newObj.put("flag", "未获取周边POI")
conn136.insert(newObj)
}
static def listAllType() {
StringBuilder ret = new StringBuilder()
for (def val : KeywordSearchUtils.KEYWORDS_MAP.values()) {
ret.append(val).append("|")
}
String result = ret.toString()
return result.substring(0, result.length() - 1)
}
}
@@ -0,0 +1,38 @@
{
"id": "B0FFF9RJ2O",
"parent": [],
"childtype": [],
"name": "丁兰广场",
"type": "购物服务;商场;购物中心",
"typecode": "060101",
"biz_type": [],
"address": "惠兰雅路与明珠街交汇处东北角",
"location": "120.231368,30.353127",
"tel": "0571-88111223",
"pname": "浙江省",
"cityname": "杭州市",
"adname": "上城区",
"importance": [],
"shopid": [],
"shopinfo": "0",
"poiweight": [],
"distance": "37",
"biz_ext": {
"rating": [],
"cost": "13953.00"
},
"photos": [
{
"title": [],
"url": "http://store.is.autonavi.com/showpic/f5dcc118938358031b45dea7e04faf45"
},
{
"title": [],
"url": "http://store.is.autonavi.com/showpic/9a0fd6ceec63e030eac3ee14055206b5"
},
{
"title": [],
"url": "http://store.is.autonavi.com/showpic/d9d2a87de3f00294665a2caeb3ee226f"
}
]
}
@@ -1,12 +1,12 @@
package com.fangjia.script.temp.shenjian.util
import com.alibaba.fastjson.JSONArray
import com.alibaba.fastjson.JSONObject
import com.fangjia.core.util.DataUtil
import com.fangjia.script.statistics.poi.AmapPoi
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.HttpRequestUtil
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import net.sf.json.JSONObject
import java.util.concurrent.ConcurrentHashMap
/**
* 高德工具类
@@ -16,7 +16,8 @@ import java.util.concurrent.ConcurrentHashMap
*/
class GaoDeUtils {
def static GAO_DE_AK = "2a1fb75990f2b39f6221e50892dd3988"
// 新获取key
def static GAO_DE_AK = "4e9592208336433ee1b7aa4d1d434418"
def static codeNameMap = new HashMap<String, String>(){{
put("150500", "地铁");
@@ -54,7 +55,7 @@ class GaoDeUtils {
* @param types "150500|1507|090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211|1601|1603|0601"; @see com.fangjia.script.wenyan.auto.KeywordSearchUtils.KEYWORDS_MAP.get(kw)
* @return
*/
static List searchNearByPoi(String city, String types, Double lng, Double lat, String radius, String keywords){
static List searchNearByPoi(String city, String types, Double lng, Double lat, Integer radius, String keywords){
def params = [:]
params.key = GAO_DE_AK
@@ -64,7 +65,7 @@ class GaoDeUtils {
params.keywords = keywords
}
params.city = city
params.radius = radius
params.radius = String.valueOf(radius)
params.location = "${lng},${lat}".toString()
List result = []
@@ -80,6 +81,63 @@ class GaoDeUtils {
if (json && json.info && json.info == "OK") {
count = json.getString("count").toInteger()
result.addAll(DataUtil.toMongoDbObj(json.pois))
} else if (json && json.infocode && json.infocode == 10003) {
throw new RuntimeException("开发者的日访问量超限,被系统自动封停,第二天0:00会自动解封")
} else if (json && json.infocode && json.infocode == 10004) {
// 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。
Thread.sleep(60000)
break;
} else {
break
}
}
return result
}
/**
* 查找周边配套
* @param city
* @param kw
* @param lng
* @param lat
* @param radius
* @param types
* @return
*/
static List searchNearDtoByPoi(String city, String types, Double lng, Double lat, Integer radius, String keywords){
def params = [:]
params.key = GAO_DE_AK
if (types) {
params.types = types
} else {
params.keywords = keywords
}
params.city = city
params.radius = String.valueOf(radius)
params.location = "${lng},${lat}".toString()
List result = []
int page = 1
int count = 10000
while (result.size() < count) {
params.page = "${page}".toString()
page++
JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params);
if (json == null) {
json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params);
}
if (json && json.info && json.info == "OK") {
count = json.getString("count").toInteger()
JSONArray jsonArray = json.pois
List<AmapPoi> ret = JSONObject.parseArray(jsonArray.toJSONString(), AmapPoi.class);
result.addAll(ret)
} else if (json && json.infocode && json.infocode == 10003) {
throw new RuntimeException("开发者的日访问量超限,被系统自动封停,第二天0:00会自动解封")
} else if (json && json.infocode && json.infocode == 10004) {
// 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。
Thread.sleep(60000)
break;
} else {
break
}
@@ -0,0 +1,29 @@
package com.fangjia.script.temp.shenjian.util
import org.apache.commons.httpclient.HttpClient
import org.apache.commons.httpclient.methods.PostMethod
import org.apache.commons.httpclient.methods.StringRequestEntity
/**
*
* @author shenjian
* @since 2021/5/25
*/
class HttpUtils {
static boolean postForJson(String url, String jsonString) {
HttpClient httpClient = new HttpClient()
StringRequestEntity requestEntity = new StringRequestEntity(
jsonString,
"application/json",
"UTF-8");
PostMethod postMethod = new PostMethod(url)
postMethod.setRequestEntity(requestEntity);
int statusCode = httpClient.executeMethod(postMethod);
if (statusCode == 200) {
return true
}
return false
}
}
@@ -176,6 +176,14 @@ class SeaweedUtils {
return ""
}
/**
* 格式化经纬度
*/
static def formatLatLng(def latOrLng) {
// 107.109878,29.192381
return com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, latOrLng / 10000000000d)
}
static void main(String[] args) {
def alias = ["金杏一村","金杏一村","金杏一村","金杏一村","金杏一村", "金杏二村"]
println(formatAlias(alias))
@@ -204,6 +204,7 @@ public class KeywordSearchUtils{
put("旅游景点","110000");
put("体育场馆","0800|0801");
put("图书馆","140500");
put("住宅区","120300|120301|120302|120303|120304");
}};
@@ -0,0 +1,71 @@
package com.fangjia.script.statistics.poi;
import java.util.List;
/**
* @author shenjian
* @since 2021/5/25
*/
@lombok.NoArgsConstructor
@lombok.Data
public class AmapPoi {
@com.alibaba.fastjson.annotation.JSONField(name = "id")
private String id;
@com.alibaba.fastjson.annotation.JSONField(name = "parent")
private List<?> parent;
@com.alibaba.fastjson.annotation.JSONField(name = "childtype")
private List<?> childtype;
@com.alibaba.fastjson.annotation.JSONField(name = "name")
private String name;
@com.alibaba.fastjson.annotation.JSONField(name = "type")
private String type;
@com.alibaba.fastjson.annotation.JSONField(name = "typecode")
private String typecode;
@com.alibaba.fastjson.annotation.JSONField(name = "biz_type")
private List<?> bizType;
@com.alibaba.fastjson.annotation.JSONField(name = "address")
private String address;
@com.alibaba.fastjson.annotation.JSONField(name = "location")
private String location;
@com.alibaba.fastjson.annotation.JSONField(name = "tel")
private String tel;
@com.alibaba.fastjson.annotation.JSONField(name = "pname")
private String pname;
@com.alibaba.fastjson.annotation.JSONField(name = "cityname")
private String cityname;
@com.alibaba.fastjson.annotation.JSONField(name = "adname")
private String adname;
@com.alibaba.fastjson.annotation.JSONField(name = "importance")
private List<?> importance;
@com.alibaba.fastjson.annotation.JSONField(name = "shopid")
private List<?> shopid;
@com.alibaba.fastjson.annotation.JSONField(name = "shopinfo")
private String shopinfo;
@com.alibaba.fastjson.annotation.JSONField(name = "poiweight")
private List<?> poiweight;
@com.alibaba.fastjson.annotation.JSONField(name = "distance")
private String distance;
@com.alibaba.fastjson.annotation.JSONField(name = "biz_ext")
private BizExtDTO bizExt;
@com.alibaba.fastjson.annotation.JSONField(name = "photos")
private List<PhotosDTO> photos;
@lombok.NoArgsConstructor
@lombok.Data
public static class BizExtDTO {
@com.alibaba.fastjson.annotation.JSONField(name = "rating")
private List<?> rating;
@com.alibaba.fastjson.annotation.JSONField(name = "cost")
private String cost;
}
@lombok.NoArgsConstructor
@lombok.Data
public static class PhotosDTO {
@com.alibaba.fastjson.annotation.JSONField(name = "title")
private List<?> title;
@com.alibaba.fastjson.annotation.JSONField(name = "url")
private String url;
}
}