新增周边POI搜索定时任务

This commit is contained in:
沈健
2021-06-18 18:08:24 +08:00
parent a93ab46f6a
commit 6d6c7cab29
6 changed files with 680 additions and 1 deletions
+6 -1
View File
@@ -482,7 +482,12 @@
<artifactId>j2v8_win32_x86_64</artifactId>
<version>4.6.0</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.18.20</version>
<scope>provided</scope>
</dependency>
</dependencies>
<repositories>
<repository>
@@ -0,0 +1,185 @@
package com.fangjia.script.temp.shenjian.poi
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.statistics.poi.AmapPoi
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.temp.shenjian.util.HttpUtils
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.apache.commons.collections.CollectionUtils
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
/**
* 小区周边POI数据获取
*
* @author shenjian
* @since 2021/5/24
*/
@BatchOwner(Watcher.ShenJian)
class SeaweedAroundPoi extends BaseBatch {
static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017)
static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi"
static ExecutorService executorService = Executors.newFixedThreadPool(30)
static void main(String[] args) {
def types = listAllType()
//dealSpecialSeaweedAround(types)
dealPoi(new BasicDBObject(), types)
executorService.shutdown()
}
/**
* 有限处理招标城市
*/
static dealSpecialSeaweedAround(def types) {
def cities = ["重庆", "达州", "曲靖", "昆明", "大理", "迪庆州", "福安", "平潭", "三明", "石狮", "柳州", "苏州"]
for (def city : cities) {
DBObject q = new BasicDBObject();
q.put("city", city);
if (city == "福安") {
city = "宁德"
q.put("city", city)
q.put("region", "福安")
} else if (city == "平潭") {
city = "福州"
q.put("city", city)
q.put("region", "平潭")
} else if (city == "石狮") {
city = "泉州"
q.put("city", city)
q.put("region", "石狮")
}
dealPoi(q, types)
}
}
static dealPoi(DBObject query, def types) {
query.put("status", 1)
conn136.find(query).batchSize(5000).each {objTemp ->
final def obj = objTemp
executorService.submit(new Runnable() {
@Override
void run() {
def city = obj.city
def lng = obj.lng
def lat = obj.lat
List<AmapPoi> pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "")
if (CollectionUtils.isNotEmpty(pois)) {
String jsonStr = JSONObject.toJSONString(pois)
HttpUtils.postForJson(URL, jsonStr)
conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI")))
} else {
conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空")))
}
println("已处理小区:" + obj)
}
})
}
}
static getNewSeaweed() {
DBObject query = new BasicDBObject()
query.put("cat", "district")
query.put("visible", 0)
//query.put("_id", new ObjectId("50c96f82e4b0e15299cb018a"))
DBObject filter = new BasicDBObject()
filter.put("_id", 1)
filter.put("city", 1)
filter.put("region", 1)
filter.put("name", 1)
filter.put("lng", 1)
filter.put("lat", 1)
List list = []
DbUtils.seaweed.find(query, filter).batchSize(10000).each {obj ->
if (conn136.count(new BasicDBObject("_id", obj._id)) > 0) {
// 已存在小区,无需同步
} else {
DBObject newObj = new BasicDBObject()
newObj.put("_id", obj._id)
newObj.put("city", obj.city)
newObj.put("region", obj.region)
newObj.put("name", obj.name)
try {
newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng))
newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat))
} catch (Exception ex) {
print("经纬度为空:" + obj._id + " " + obj)
ex.printStackTrace()
}
newObj.put("status", 1)
newObj.put("flag", "未获取周边POI")
list << newObj
if (list.size() == 5000) {
conn136.insert(list)
list.clear()
}
}
}
conn136.insert(list)
list.clear()
}
static def getByIdAndInsert(String id) {
BasicDBObject obj = DbUtils.seaweed.findById(id);
DBObject newObj = new BasicDBObject()
newObj.put("_id", obj._id)
newObj.put("city", obj.city)
newObj.put("region", obj.region)
newObj.put("name", obj.name)
try {
newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng))
newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat))
} catch (Exception ex) {
print("经纬度为空:" + obj._id + " " + obj)
ex.printStackTrace()
}
newObj.put("status", 1)
newObj.put("flag", "未获取周边POI")
conn136.insert(newObj)
}
static def listAllType() {
StringBuilder ret = new StringBuilder()
for (def val : KEYWORDS_MAP.values()) {
ret.append(val).append("|")
}
String result = ret.toString()
return result.substring(0, result.length() - 1)
}
public static final Map<String,String> KEYWORDS_MAP = new HashMap<String, String>(){{
put("地铁", "150500");
put("公交", "1507");
put("医院", "090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211");
put("写字楼", "120201");
put("学校","141200|141201|141202|141203|141204");
put("银行","1601|1603");
put("大型超市","0604");
put("商场","0601");
put("餐饮","0501|0502|0503|0504");
put("旅游景点","110000");
put("图书馆","140500");
}};
@Override
void process(Object args) {
def types = listAllType()
//dealSpecialSeaweedAround(types)
dealPoi(new BasicDBObject(), types)
executorService.shutdown()
}
}
@@ -0,0 +1,181 @@
package com.fangjia.script.temp.shenjian.util
import com.alibaba.fastjson.JSONArray
import com.alibaba.fastjson.JSONObject
import com.fangjia.core.util.DataUtil
import com.fangjia.script.statistics.poi.AmapPoi
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.HttpRequestUtil
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import java.util.concurrent.atomic.AtomicInteger
/**
* 高德工具类
*
* @author shenjian
* @since 2021/5/15
*/
class GaoDeUtils {
// 新获取key
def static GAO_DE_AK = ["4e9592208336433ee1b7aa4d1d434418", "35925efd7333f1df52145e51b8e4e56a"]
def static AtomicInteger keyUsed = new AtomicInteger(0)
def static codeNameMap = new HashMap<String, String>(){{
put("150500", "地铁");
put("1507", "公交");
put("090100", "医院")
put("090101", "医院")
put("090200", "医院")
put("090202", "医院")
put("090202", "医院")
put("090204", "医院")
put("090205", "医院")
put("090206", "医院")
put("090206", "医院")
put("090208", "医院")
put("090209", "医院")
put("090210", "医院")
put("090211", "医院")
put("141200", "学校")
put("141201", "学校")
put("141202", "学校")
put("141203", "学校")
put("141204", "学校")
put("1601", "银行")
put("1603", "银行")
put("0601", "商场")
}};
/**
* 查找周边配套
* @param city
* @param kw
* @param lng
* @param lat
* @param radius
* @param types "150500|1507|090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211|1601|1603|0601"; @see com.fangjia.script.wenyan.auto.KeywordSearchUtils.KEYWORDS_MAP.get(kw)
* @return
*/
static List searchNearByPoi(String city, String types, Double lng, Double lat, Integer radius, String keywords){
def params = [:]
params.key = GAO_DE_AK[keyUsed]
if (types) {
params.types = types
} else {
params.keywords = keywords
}
params.city = city
params.radius = String.valueOf(radius)
params.location = "${lng},${lat}".toString()
List result = []
int page = 1
int count = 10000
while (result.size() < count) {
params.page = "${page}".toString()
page++
JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params);
if (json == null) {
json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params);
}
if (json && json.info && json.info == "OK") {
count = json.getString("count").toInteger()
result.addAll(DataUtil.toMongoDbObj(json.pois))
} else if (json && json.infocode && json.infocode == 10003) {
throw new RuntimeException("开发者的日访问量超限,被系统自动封停,第二天0:00会自动解封")
} else if (json && json.infocode && json.infocode == 10004) {
// 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。
Thread.sleep(60000)
break;
} else {
break
}
}
return result
}
/**
* 查找周边配套
* @param city
* @param kw
* @param lng
* @param lat
* @param radius
* @param types
* @return
*/
static List searchNearDtoByPoi(String city, String types, Double lng, Double lat, Integer radius, String keywords){
def params = [:]
params.key = GAO_DE_AK[keyUsed.intValue()]
if (types) {
params.types = types
} else {
params.keywords = keywords
}
params.city = city
params.radius = String.valueOf(radius)
params.location = "${lng},${lat}".toString()
List result = []
int page = 1
int count = 10000
while (result.size() < count) {
params.page = "${page}".toString()
page++
JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params);
if (json == null) {
json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params);
}
if (json && json.info && json.info == "OK") {
count = json.getString("count").toInteger()
try {
List<AmapPoi> ret = JSONObject.parseArray(json.pois.toString(), AmapPoi.class)
result.addAll(ret)
} catch(Exception ex) {
ex.printStackTrace()
}
} else if (json && json.infocode && json.infocode == "10044") {
//throw new RuntimeException("开发者的日访问量超限,被系统自动封停,第二天0:00会自动解封")
// 多线程情况下还是exit有效果,抛异常不会停止程序
if (keyUsed.intValue() >= GAO_DE_AK.size()) {
System.exit(1)
}
keyUsed.incrementAndGet()
} else if (json && json.infocode && json.infocode == "10004") {
// 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。
Thread.sleep(60000)
break;
} else {
break
}
}
return result
}
/**
* 获取经纬度之间距离
*/
static def getDistance(double Lat, double Lng, double Lat1, double Lng1) {
return GeoHelper.GetDistance(Lat, Lng, Lat1, Lng1)
}
/**
* 获取地址经纬度
*/
static def parserByGaode(def city, def address) {
JSONObject json = GaoDeApiUtils.geo(city, address)
if (json && json.status == "1") {
def ret = json.geocodes
if (ret) {
def str = ret.get(0).location
def level = ret.get(0).level
def arrs = str.split(",")
return ["lng": arrs[0], "lat": arrs[1]]
}
}
}
}
@@ -0,0 +1,29 @@
package com.fangjia.script.temp.shenjian.util
import org.apache.commons.httpclient.HttpClient
import org.apache.commons.httpclient.methods.PostMethod
import org.apache.commons.httpclient.methods.StringRequestEntity
/**
*
* @author shenjian
* @since 2021/5/25
*/
class HttpUtils {
static boolean postForJson(String url, String jsonString) {
HttpClient httpClient = new HttpClient()
StringRequestEntity requestEntity = new StringRequestEntity(
jsonString,
"application/json",
"UTF-8");
PostMethod postMethod = new PostMethod(url)
postMethod.setRequestEntity(requestEntity);
int statusCode = httpClient.executeMethod(postMethod);
if (statusCode == 200) {
return true
}
return false
}
}
@@ -0,0 +1,208 @@
package com.fangjia.script.temp.shenjian.util
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.datastore.FieldValidatorUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.elasticsearch.geometry.utils.Geohash
import java.util.concurrent.ConcurrentHashMap
/**
* 小区工具类
*
* @author shenjian
* @since 2021/5/17
*/
class SeaweedUtils {
static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017)
static def map = new ConcurrentHashMap()
static getRegionByCode(def district_code) {
def region;
if (map.get("district_code") != null) {
region = map.get("district_code")
} else {
cityMapConn.find(new BasicDBObject("district_code", district_code)).each { obj ->
region = obj.fangjia_district
map.put("district_code", region)
}
}
return region
}
/**
* 标准化输出绿化率
*/
static formatVirescencePercent(def virescence_percent) {
// 绿化率
if (virescence_percent && FieldValidatorUtils.checkValue(virescence_percent)) {
double val = Double.valueOf(virescence_percent)
if (val < 1) {
return val * 100
}
if (val == 1) {
return val * 10
}
if (val >= 70) {
return 65
}
return val
}
return ""
}
/**
* 标准化容积率输出
*/
static formatCubagePercent(def cubage_percent) {
if (cubage_percent && FieldValidatorUtils.checkValue(cubage_percent)) {
double val = Double.valueOf(cubage_percent)
if (val >= 7 && val < 8) {
return 6.2
}
if (val >= 8 && val < 9) {
return 6.3
}
if (val >= 9 && val < 10) {
return 6.5
}
if (val >= 10) {
return 6.7
}
return val
}
return ""
}
/**
* 标准化车位输出
*/
static formatCarSiteCount(def car_site_count) {
if (car_site_count && FieldValidatorUtils.checkValue(car_site_count)) {
long val = Long.valueOf(car_site_count)
if (val > 2000 || val < 100) {
return ""
}
return val;
}
return ""
}
/**
* 标准化建筑面积输出
*/
static formatBuildArea(def build_area) {
if (build_area && FieldValidatorUtils.checkValue(build_area)) {
double val = build_area
if (val > 1000) {
return val
}
}
return ""
}
/**
* 标准化总占地面积输出
*/
static formatAllArea(def all_area) {
if (all_area) {
Double formattedVal = 0.0;
if (all_area.toString().indexOf("公顷") != -1) {
def val = all_area.toString()
formattedVal = (Double.valueOf(val.substring(0, val.indexOf("公顷"))) * 10000)
} else if(FieldValidatorUtils.checkValue(all_area)) {
formattedVal = Double.valueOf(all_area)
}
if (formattedVal > 100000 || formattedVal == 0.0) {
return ""
}
return formattedVal
}
}
/**
* 标准化户数输出
*/
static formatHouseholdCount(def household_count) {
if (household_count && FieldValidatorUtils.checkValue(household_count) ) {
if (Long.valueOf(household_count) < 50) {
return "" // 户数
}
return household_count
}
}
/**
* 计算车位比
*/
static calcCarAndHouseholdRadio(def car_site_count, def household_count) {
if (SeaweedUtils.formatCarSiteCount(car_site_count) && SeaweedUtils.formatHouseholdCount(household_count)) {
long formattedCarSiteCount = SeaweedUtils.formatCarSiteCount(car_site_count)
long formattedHouseholdCount = SeaweedUtils.formatHouseholdCount(household_count)
double val = DataHandleUtils.getDoubleRound(1, (Double.valueOf(formattedCarSiteCount) / Double.valueOf(formattedHouseholdCount)))
if (val == 0.0) {
val = DataHandleUtils.getDoubleRound(2, (Double.valueOf(formattedCarSiteCount) / Double.valueOf(formattedHouseholdCount)))
}
if (val == 0.0) {
val = DataHandleUtils.getDoubleRound(3, (Double.valueOf(formattedCarSiteCount) / Double.valueOf(formattedHouseholdCount)))
}
if (val == 1.0) {
return "1:1"
}
return "1:" + val
}
return ""
}
/**
* 标准化别名输出
*/
static formatAlias(def alias) {
if (alias) {
// 小区可能存在多个别名
def temp = []
int len = 0
for (def a : alias) {
len += a.toString().length()
if (len > 20) {
return temp.join(",")
}
temp.add(a)
}
return temp.join(",")
}
return ""
}
/**
* 格式化经纬度
*/
static def formatLatLng(def latOrLng) {
// 107.109878,29.192381
return com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, latOrLng / 10000000000d)
}
/**
* 小区经纬度geohash编码
* @param args
*/
static def encodeByGeoHash(double lon, double lat, int level=7) {
return Geohash.stringEncode(lon, lat, level)
}
/**
* 小区经纬度geohash解码
* @param args
*/
static def decodeByGeoHash(String geohash) {
return Geohash.decodeLongitude(geohash) + "," + Geohash.decodeLatitude(geohash)
}
static void main(String[] args) {
def alias = ["金杏一村","金杏一村","金杏一村","金杏一村","金杏一村", "金杏二村"]
println(formatAlias(alias))
}
}
@@ -0,0 +1,71 @@
package com.fangjia.script.statistics.poi;
import java.util.List;
/**
* @author shenjian
* @since 2021/5/25
*/
@lombok.NoArgsConstructor
@lombok.Data
public class AmapPoi {
@com.alibaba.fastjson.annotation.JSONField(name = "id")
private String id;
@com.alibaba.fastjson.annotation.JSONField(name = "parent")
private List<?> parent;
@com.alibaba.fastjson.annotation.JSONField(name = "childtype")
private List<?> childtype;
@com.alibaba.fastjson.annotation.JSONField(name = "name")
private String name;
@com.alibaba.fastjson.annotation.JSONField(name = "type")
private String type;
@com.alibaba.fastjson.annotation.JSONField(name = "typecode")
private String typecode;
@com.alibaba.fastjson.annotation.JSONField(name = "biz_type")
private List<?> bizType;
@com.alibaba.fastjson.annotation.JSONField(name = "address")
private String address;
@com.alibaba.fastjson.annotation.JSONField(name = "location")
private String location;
@com.alibaba.fastjson.annotation.JSONField(name = "tel")
private String tel;
@com.alibaba.fastjson.annotation.JSONField(name = "pname")
private String pname;
@com.alibaba.fastjson.annotation.JSONField(name = "cityname")
private String cityname;
@com.alibaba.fastjson.annotation.JSONField(name = "adname")
private String adname;
@com.alibaba.fastjson.annotation.JSONField(name = "importance")
private List<?> importance;
@com.alibaba.fastjson.annotation.JSONField(name = "shopid")
private List<?> shopid;
@com.alibaba.fastjson.annotation.JSONField(name = "shopinfo")
private String shopinfo;
@com.alibaba.fastjson.annotation.JSONField(name = "poiweight")
private List<?> poiweight;
@com.alibaba.fastjson.annotation.JSONField(name = "distance")
private String distance;
@com.alibaba.fastjson.annotation.JSONField(name = "biz_ext")
private BizExtDTO bizExt;
@com.alibaba.fastjson.annotation.JSONField(name = "photos")
private List<PhotosDTO> photos;
@lombok.NoArgsConstructor
@lombok.Data
public static class BizExtDTO {
@com.alibaba.fastjson.annotation.JSONField(name = "rating")
private List<?> rating;
@com.alibaba.fastjson.annotation.JSONField(name = "cost")
private String cost;
}
@lombok.NoArgsConstructor
@lombok.Data
public static class PhotosDTO {
@com.alibaba.fastjson.annotation.JSONField(name = "title")
private List<?> title;
@com.alibaba.fastjson.annotation.JSONField(name = "url")
private String url;
}
}