新增小区高德搜索

This commit is contained in:
Jian Shen
2022-05-25 16:43:10 +08:00
parent 4f2e84a5d9
commit d57955ef1e
3 changed files with 259 additions and 0 deletions
@@ -0,0 +1,157 @@
package com.fangjia.script.temp.shenjian.poi
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.statistics.poi.AmapPoi
import com.fangjia.script.statistics.poi.AmapPoiDetail
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.temp.shenjian.util.HttpUtils
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.wenyan.auto.KeywordSearchUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.client.model.Filters
import com.mongodb.client.model.UpdateOneModel
import com.mongodb.client.model.UpdateOptions
import com.mongodb.client.model.WriteModel
import org.apache.commons.collections.CollectionUtils
import org.apache.commons.lang.StringUtils
import org.bson.Document
import org.bson.conversions.Bson
import org.bson.types.ObjectId
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
/**
* 小区POI信息精确查找
*
* @author shenjian
* @since 2022/5/25
*/
class SeaweedPoiDetail {
static def conn136 = MongoUtils.getCollectionByName3("fangjia", "seaweed_poi_sync", "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017")
static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoiDetail"
// 住宅类
static final String typecodes = "120000|120100|120200|120201|120202|120203|120300|120301|120302|120303|120304";
static final ExecutorService executorService = Executors.newFixedThreadPool(20)
static void main(String[] args) {
DBObject query = new BasicDBObject()
query.put("status", 1)
query.put("city", "东莞")
conn136.find(query).batchSize(1000).each {objTemp ->
final def obj = objTemp
executorService.submit(new Runnable() {
@Override
void run() {
def city = obj.city
def region = obj.region
def name = obj.name
def alias = obj.alias
def address = obj.address
def addr_alias = obj.addr_alias
StringJoiner stringJoiner = new StringJoiner("|")
// 主小区名搜索
stringJoiner.add(name)
// 根据别名搜
if (alias != null && CollectionUtils.isNotEmpty(alias)) {
for (String str : alias) {
stringJoiner.add(str)
}
}
// 根据路名搜
if (StringUtils.isNotBlank(address)) {
stringJoiner.add(address)
}
// 根据路别名搜
if (addr_alias != null && CollectionUtils.isNotEmpty(addr_alias)) {
for (String str : addr_alias) {
stringJoiner.add(str)
}
}
String keywords = stringJoiner.toString()
List pois = GaoDeUtils.searchPoi(city, keywords, region, typecodes)
List<AmapPoiDetail> detailList = new ArrayList<>()
for (AmapPoi amapPoi : pois) {
List<AmapPoiDetail> amapPoiList = GaoDeUtils.getAmapDetailById(amapPoi.getId());
if (CollectionUtils.isNotEmpty(amapPoiList)) {
detailList.addAll(amapPoiList)
}
}
if (CollectionUtils.isNotEmpty(detailList)) {
String jsonStr = JSONObject.toJSONString(detailList)
HttpUtils.postForJson(URL, jsonStr)
conn136.updateOne(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取到")))
} else {
conn136.updateOne(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "未获取到小区")))
}
println("已处理小区:" + obj)
}
})
}
executorService.shutdown()
}
static getNewSeaweed() {
DBObject query = new BasicDBObject()
query.put("cat", "district")
query.put("visible", 0)
DBObject filter = new BasicDBObject()
filter.put("_id", 1)
filter.put("city", 1)
filter.put("region", 1)
filter.put("name", 1)
filter.put("alias", 1)
filter.put("address", 1)
filter.put("addr_alias", 1)
filter.put("lng", 1)
filter.put("lat", 1)
List<WriteModel<Document>> writeModelList = new ArrayList<>()
def count = 0
DbUtils.seaweed.find(query, filter).batchSize(10000).each {obj ->
Document updateBson = new Document()
Document update = new Document()
update.put("city", obj.city)
update.put("region", obj.region)
update.put("name", obj.name)
update.put("alias", obj.alias)
update.put("address", obj.address)
update.put("addr_alias", obj.addr_alias)
try {
update.put("lng", SeaweedUtils.formatLatLng(obj.lng))
update.put("lat", SeaweedUtils.formatLatLng(obj.lat))
} catch (Exception ex) {
print("经纬度为空:" + obj._id + " " + obj)
}
update.put("status", 1)
update.put("flag", "未精确获取高德信息")
updateBson.append('$set', update)
Bson filterBson = Filters.eq("_id", obj._id)
UpdateOptions updateOptions = new UpdateOptions();
updateOptions.upsert(true)
UpdateOneModel updateOneModel = new UpdateOneModel(filterBson, updateBson, updateOptions);
writeModelList.add(updateOneModel)
if (writeModelList.size() == 5000) {
conn136.bulkWrite(writeModelList)
count += 5000
println(count)
writeModelList.clear()
}
}
if (writeModelList.size() > 0) {
conn136.bulkWrite(writeModelList)
writeModelList.clear()
}
}
}
@@ -7,10 +7,12 @@ import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.statistics.poi.AmapPoi
import com.fangjia.script.statistics.poi.AmapPoiDetail
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.HttpRequestUtil
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import org.apache.commons.lang.StringUtils
import org.openqa.selenium.json.Json
import java.util.concurrent.atomic.AtomicInteger
@@ -232,6 +234,35 @@ class GaoDeUtils {
return Collections.emptyList()
}
/**
* 获取详情
*/
static List<AmapPoiDetail> getAmapDetailById(id){
def params = [:]
params.key = GAO_DE_AK[keyUsed.intValue()]
params.id = id
params.output = 'json'
JSONObject json = HttpRequestUtil.getJsonContent("https://restapi.amap.com/v3/place/detail", params);
if (json == null) {
json = HttpRequestUtil.getJsonContent("https://restapi.amap.com/v3/place/detail", params);
}
if (json && json.info && json.info == "OK") {
try {
List<AmapPoiDetail> ret = JSONObject.parseArray(json.pois.toString(), AmapPoi.class)
return ret
} catch(Exception ex) {
ex.printStackTrace()
}
} else if (json && json.infocode && json.infocode == "10004") {
// 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。
Thread.sleep(60000)
} else if (json && json.infocode && json.infocode == "10017") {
// 所请求的资源不可用
System.exit(1)
}
return Collections.emptyList()
}
/**
* 查找关键字经纬度
*/
@@ -0,0 +1,71 @@
package com.fangjia.script.statistics.poi;
import java.util.List;
/**
* @author shenjian
* @since 2022/5/25
*/
@lombok.NoArgsConstructor
@lombok.Data
public class AmapPoiDetail {
@com.alibaba.fastjson.annotation.JSONField(name = "id")
private String id;
@com.alibaba.fastjson.annotation.JSONField(name = "parent")
private List<?> parent;
@com.alibaba.fastjson.annotation.JSONField(name = "childtype")
private List<?> childtype;
@com.alibaba.fastjson.annotation.JSONField(name = "name")
private String name;
@com.alibaba.fastjson.annotation.JSONField(name = "type")
private String type;
@com.alibaba.fastjson.annotation.JSONField(name = "typecode")
private String typecode;
@com.alibaba.fastjson.annotation.JSONField(name = "biz_type")
private List<?> bizType;
@com.alibaba.fastjson.annotation.JSONField(name = "address")
private String address;
@com.alibaba.fastjson.annotation.JSONField(name = "location")
private String location;
@com.alibaba.fastjson.annotation.JSONField(name = "tel")
private String tel;
@com.alibaba.fastjson.annotation.JSONField(name = "pname")
private String pname;
@com.alibaba.fastjson.annotation.JSONField(name = "cityname")
private String cityname;
@com.alibaba.fastjson.annotation.JSONField(name = "adname")
private String adname;
@com.alibaba.fastjson.annotation.JSONField(name = "importance")
private List<?> importance;
@com.alibaba.fastjson.annotation.JSONField(name = "shopid")
private List<?> shopid;
@com.alibaba.fastjson.annotation.JSONField(name = "shopinfo")
private String shopinfo;
@com.alibaba.fastjson.annotation.JSONField(name = "poiweight")
private List<?> poiweight;
@com.alibaba.fastjson.annotation.JSONField(name = "distance")
private String distance;
@com.alibaba.fastjson.annotation.JSONField(name = "biz_ext")
private BizExtDTO bizExt;
@com.alibaba.fastjson.annotation.JSONField(name = "photos")
private List<PhotosDTO> photos;
@lombok.NoArgsConstructor
@lombok.Data
public static class BizExtDTO {
@com.alibaba.fastjson.annotation.JSONField(name = "rating")
private List<?> rating;
@com.alibaba.fastjson.annotation.JSONField(name = "cost")
private String cost;
}
@lombok.NoArgsConstructor
@lombok.Data
public static class PhotosDTO {
@com.alibaba.fastjson.annotation.JSONField(name = "title")
private List<?> title;
@com.alibaba.fastjson.annotation.JSONField(name = "url")
private String url;
}
}