跑高德地图

This commit is contained in:
2025-02-20 11:44:45 +08:00
parent 493e4c3433
commit 26591c2d22
3 changed files with 230 additions and 63 deletions
@@ -4,11 +4,16 @@ import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.BaiduMapUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import org.apache.commons.io.FileUtils
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.action.support.WriteRequest
import org.elasticsearch.action.update.UpdateRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.common.xcontent.XContentType
@@ -31,7 +36,7 @@ class TownStandardBatch extends BaseBatch{
def fullCityMap = CommonDatautils.queryFullNameCity()
def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity()
def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity()
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(100)
def updateList = []
def addAliasList = []
@@ -49,7 +54,8 @@ class TownStandardBatch extends BaseBatch{
def TownStandard(){
// townStandardByCat("office")
// townStandardByPoi()
addTownByTownShape()
addTownByTownShape()//为没有乡镇街道的poi,补充乡镇街道,忽略区域,并使用地图
// updateTownData()
}
def townStandardByCat(def cat){
//"三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德",
@@ -385,11 +391,41 @@ class TownStandardBatch extends BaseBatch{
return rets;
}
def readCSV3(filename){
def lines = FileUtils.readLines(new File(filename))
def rets = [];
def index = 0
for(line in lines) {
if(line && index>=1){
String[] arrs = line.split(",")
if(arrs.size()>=10 && arrs[7] && arrs[9]=="同区域"){
def o = new BasicDBObject()
try{
o.put("id", arrs[0])
o.put("township", arrs[7])
if(arrs.size()>=9 && arrs[8]){
o.put("district", arrs[8])
}
if(arrs.size()>=10 && arrs[9]){
o.put("way", arrs[9])
}
rets << o
}catch(Exception e){
println arrs
}
}
}
index++
}
return rets;
}
def addTownByTownShape(){
def i = 0
def headers = ["_id","province","city","district","lng","lat","data_source","乡镇街道", "新区域", "方式"]
def headers = ["_id","province","city","district","lng","lat","data_source","乡镇街道", "新区域", "方式", "标准乡镇"]
while(i<50){
def fileName = "/home/goojia/gj/new_poi/input/no_township_${i}.csv"
def fileName = "/home/goojia/gj/new_poi/input/map/no_township_${i}.csv"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\数据建设\\乡镇街道\\待补充乡镇街道\\原始\\no_township_${i}.csv"
}
@@ -398,7 +434,7 @@ class TownStandardBatch extends BaseBatch{
def rets = readCSV2(fileName)
def rows = addTownshipBatch(rets)
if (CommonDatautils.isWindow()) {
com.fangjia.script.gj.ccb.FileUtils.batchCreateFileByCustomSuffix(rows, "D:\\数据\\数据建设\\乡镇街道\\待补充乡镇街道\\结果\\no_township_result_${i}.csv", headers, "utf-8", false, ',', '');
com.fangjia.script.gj.ccb.FileUtils.batchCreateFileByCustomSuffix(rows, "D:\\数据\\数据建设\\乡镇街道\\待补充乡镇街道\\结果-地图\\no_township_result_${i}.csv", headers, "utf-8", false, ',', '');
}else{
com.fangjia.script.gj.ccb.FileUtils.batchCreateFileByCustomSuffix(rows, "/home/goojia/gj/new_poi/output/no_township_result_${i}.csv", headers, "utf-8", false, ',', '');
}
@@ -425,10 +461,10 @@ class TownStandardBatch extends BaseBatch{
def townMap = CommonDatautils.queryTownByCity(r.city, ["city", "district", "mzb_district_name", "street", "alias", "shape"])
def set = rMap.keySet()
println totalCount.get() +" "+r.city + " " + set.size()
if (set.size() >= 1) {
// if (set.size() >= 1) {
addTown(r, townMap, ["lon":r.lng.toString().toDouble(), "lat":r.lat.toString().toDouble()])
results << [r.id, r.province, r.city, r.district, r.lng, r.lat, r.data_source, r.township, r.new_district, r.town_source]
}
results << [r.id, r.province, r.city, r.district, r.lng, r.lat, r.data_source, r.township, r.new_district, r.town_source, r.strand_town]
// }
}
} catch (Exception e) {
e.printStackTrace()
@@ -444,52 +480,87 @@ class TownStandardBatch extends BaseBatch{
def addTown(saveObj, townMap, location) {
//在poi本区域下通过乡镇街道围栏选出所在乡镇街道
def towns = townMap.get(saveObj.district)
if (towns) {
for (def town : towns) {
if (town.shape && town.shape.coordinates) {
def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates))
if (flag) {
saveObj.township = town.street
saveObj.town_source = "同区域"
break
}
}
}
}
// def towns = townMap.get(saveObj.district)
// if (towns) {
// for (def town : towns) {
// if (town.shape && town.shape.coordinates) {
// def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates))
// if (flag) {
// saveObj.township = town.street
// saveObj.town_source = "同区域"
// saveObj.strand_town = "是"
// break
// }
// }
// }
// }
//忽略poi中的区域 通过乡镇街道围栏选出所在乡镇街道
// if(!saveObj.township){
// def townNames = townMap.keySet()
// for(def townName : townNames){
// towns = townMap.get(townName)
// for (def town : towns) {
// if (town.shape && town.shape.coordinates) {
// def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates))
// if (flag) {
// saveObj.township = town.street
// saveObj.new_district = town.mzb_district_name
// saveObj.town_source = "忽略区域"
// saveObj.strand_town = isStandTown(townMap, saveObj.new_district, saveObj.township)
// break
// }
// }
// }
// if(saveObj.township){
// break;
// }
// }
// }
//通过调取地图api接口获取乡镇街道
if(!saveObj.township){
def townNames = townMap.keySet()
for(def townName : townNames){
towns = townMap.get(townName)
for (def town : towns) {
if (town.shape && town.shape.coordinates) {
def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates))
if (flag) {
saveObj.township = town.street
saveObj.new_district = town.mzb_district_name
saveObj.town_source = "忽略区域"
break
}
}
}
if(saveObj.township){
break;
def gaodeMap = getTownshipInfo(location.lon+","+location.lat)
if(gaodeMap && gaodeMap.township){
saveObj.township = gaodeMap.township
saveObj.new_district = gaodeMap.district
saveObj.town_source = "高德API"
if(townMap.size()>=1){
saveObj.strand_town = isStandTown(townMap, saveObj.new_district, saveObj.township)
}
}
}
//通过调取地图api接口获取乡镇街道
// //通过调取地图api接口获取乡镇街道
// if(!saveObj.township){
// def gaodeMap = getTownshipInfo(location.lon+","+location.lat)
// if(gaodeMap && gaodeMap.township){
// saveObj.township = gaodeMap.township
// saveObj.new_district = gaodeMap.district
// saveObj.town_source = "高德API"
// def gaodeMap = BaiduMapUtils.geocoding(location.lat+","+location.lon)
// if(gaodeMap && gaodeMap.addressComponent) {
// def addressComponent = gaodeMap.addressComponent
// if(addressComponent){
// if(addressComponent.town){
// saveObj.township = addressComponent.town
// saveObj.new_district = addressComponent.district
// saveObj.town_source = "百度API"
// saveObj.strand_town = isStandTown(townMap, saveObj.new_district, saveObj.township)
// }
// }
// }
// }
return saveObj
}
def isStandTown(townMap, district, town){
def towns = townMap.get(district)
def flag = "非"
if(towns){
for(def t : towns){
if(t.street == town){
flag = "是"
break
}
}
}
return flag
}
def getTownshipInfo(def glocation){
def township = null
@@ -498,9 +569,11 @@ class TownStandardBatch extends BaseBatch{
def format_addr = null
def stop = false
def count = 0
while(!stop){
def jsonObject = null
// while(!stop){
if(glocation){
def jsonObject = GaoDeApiUtils.geocode(glocation)
jsonObject = GaoDeApiUtils.geocode(glocation)
if(jsonObject && jsonObject.regeocode) {
def addressComponent = jsonObject.regeocode.addressComponent
if(addressComponent){
@@ -512,14 +585,15 @@ class TownStandardBatch extends BaseBatch{
}
}
}
if(district || count>10){
stop = true
}else{
count ++
Thread.sleep(3000)
print "."
}
}
// if(jsonObject.status==0 || count>3){
// stop = true
// println "高德调用超3次"
// }else{
// count ++
// Thread.sleep(3000)
// print "."
// }
// }
def map = [:]
map.city = city
@@ -529,6 +603,72 @@ class TownStandardBatch extends BaseBatch{
return map;
}
def updateTownData(){
def i = 0
while(i<50) {
def fileName = "/home/goojia/gj/new_poi/output/no_township_result_${i}.csv"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\数据建设\\乡镇街道\\待补充乡镇街道\\结果\\no_township_result_${i}.csv"
}
File file = new File(fileName);
if (file.exists()) {
def rets = readCSV3(fileName)
println file.getAbsoluteFile().getName() +" "+(rets.size()*100/200000)+"%"+" "+rets.size()
// if(rets){
// updateToEs(rets)
// }
}
i++
}
}
def updateToEs(rows){
BulkRequest request = new BulkRequest();
int batchSize = 1;
int processedCount = 0;
def client = ElasticUtils.getClient89("192.168.0.89")
for (int i = 0; i < rows.size(); i++) {
def map = rows[i]
if(map.containsKey("way")){
map.remove("way")
}
request.add(new IndexRequest("map_poi_2024", "_doc", map.id).source(map, XContentType.JSON));
if(i%1000==0){
print "."
}
// 批量提交数据
if (request.numberOfActions() >= batchSize) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
print "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents"
}
request = new BulkRequest();
}
}
if (request.numberOfActions() > 0) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents"
}
}
println "结束时间 "+new Date()
}
}
@@ -9,13 +9,8 @@ import net.sf.json.JSONObject
class BaiduMapUtils {
private static String AK = "9E99D3540e4dafb7894f7b9fe27fa6b5";
def static AKS = ["hj5KqPjaGA3uivtg8uS8XfAnj9X1kPMr","xjFg7DfDqvQleIZasM3ph8Zi2Lw8kgCg",
"4pb6fzrWIzvibDu8L1o8DGoY2h1wRIUq","ZzX6mRFfE5GKn4cpUp24Q3OyUIHnfQAR","79ac4ea49e7dedbf80b0ecd475d07784",
"9E99D3540e4dafb7894f7b9fe27fa6b5","W3M5uHfcrmlAdhQjf6v93LCI", "C8bb035aacf602551e829e4c64575224",
"B42b12d01f2dee712fd66c3e886158a4", "1a3cde429f38434f1811a75e1a90310c","W3M5uHfcrmlAdhQjf6v93LCI",
"C8bb035aacf602551e829e4c64575224", "B42b12d01f2dee712fd66c3e886158a4",
"1a3cde429f38434f1811a75e1a90310c","9E99D3540e4dafb7894f7b9fe27fa6b5"];
private static String AK = "1a3cde429f38434f1811a75e1a90310c";
def static AKS = ["1a3cde429f38434f1811a75e1a90310c","1a3cde429f38434f1811a75e1a90310c"];
public static boolean HAS_AVAILABLE_AK = true;
static def getAddress(lng, lat) throws Exception{
@@ -83,8 +78,8 @@ class BaiduMapUtils {
throw new RuntimeException("无可用百度AK");
}
System.out.println("更换AK");
for(int i = 0 ;i < AKS.length; i++){
if(i == AKS.length - 1){
for(int i = 0 ;i < AKS.size(); i++){
if(i == AKS.size() - 1){
AK = AKS[0];
HAS_AVAILABLE_AK = false;
throw new RuntimeException("无可用百度AK");
@@ -127,6 +122,35 @@ class BaiduMapUtils {
return null
}
public static JSONObject geocoding(locationstr) {
String url = "https://api.map.baidu.com/reverse_geocoding/v3/?ak="+AK+"&output=json&coordtype=gcj02ll&location="+locationstr
boolean stop = false;
int recount = 0;
JSONObject jsonObj = null
while(!stop && recount<6) {
try{
HttpRequest request = new HttpRequest();
try{
String result = request.get(url,null,"UTF-8");
def json = JSONObject.fromObject(result);
if(json.getInt("status") >= 300){
changeAK();
recount++
}else{
jsonObj = json.result
}
}catch (Exception e){
e.printStackTrace();
}
}catch (Exception e){
recount++;
println e.getMessage()
}
}
return jsonObj
}
public static def searchByBounds(key, min_lat, min_lng, max_lat, max_lng){
boolean stop = false;
int recount = 0;
@@ -12,7 +12,7 @@ import org.jsoup.Jsoup
import java.util.zip.GZIPInputStream
class GaoDeApiUtils {
def static GAO_DE_AK = "dfc2245bac8a7b2d5b62e060e59d4eca" //宝剑:c6fcd4bfbc40d73ee762b8115dca3390
def static GAO_DE_AK = "94c74c60e102972a8d17b7b75d7403be" //宝剑:c6fcd4bfbc40d73ee762b8115dca3390
//15021384995:dfc2245bac8a7b2d5b62e060e59d4eca
/**
@@ -35,6 +35,9 @@ class GaoDeApiUtils {
content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8")
}
JSONObject jsonObj = JSONObject.fromObject(content)
if(jsonObj.status!=1){
println jsonObj
}
return jsonObj
}