跑高德地图
This commit is contained in:
+193
-53
@@ -4,11 +4,16 @@ import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.BaiduMapUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.apache.commons.io.FileUtils
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
import org.elasticsearch.action.index.IndexRequest
|
||||
import org.elasticsearch.action.support.WriteRequest
|
||||
import org.elasticsearch.action.update.UpdateRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.common.xcontent.XContentType
|
||||
@@ -31,7 +36,7 @@ class TownStandardBatch extends BaseBatch{
|
||||
def fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity()
|
||||
def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity()
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(100)
|
||||
|
||||
def updateList = []
|
||||
def addAliasList = []
|
||||
@@ -49,7 +54,8 @@ class TownStandardBatch extends BaseBatch{
|
||||
def TownStandard(){
|
||||
// townStandardByCat("office")
|
||||
// townStandardByPoi()
|
||||
addTownByTownShape()
|
||||
addTownByTownShape()//为没有乡镇街道的poi,补充乡镇街道,忽略区域,并使用地图
|
||||
// updateTownData()
|
||||
}
|
||||
def townStandardByCat(def cat){
|
||||
//"三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德",
|
||||
@@ -385,11 +391,41 @@ class TownStandardBatch extends BaseBatch{
|
||||
return rets;
|
||||
}
|
||||
|
||||
def readCSV3(filename){
|
||||
def lines = FileUtils.readLines(new File(filename))
|
||||
def rets = [];
|
||||
def index = 0
|
||||
for(line in lines) {
|
||||
if(line && index>=1){
|
||||
String[] arrs = line.split(",")
|
||||
if(arrs.size()>=10 && arrs[7] && arrs[9]=="同区域"){
|
||||
def o = new BasicDBObject()
|
||||
try{
|
||||
o.put("id", arrs[0])
|
||||
o.put("township", arrs[7])
|
||||
if(arrs.size()>=9 && arrs[8]){
|
||||
o.put("district", arrs[8])
|
||||
}
|
||||
if(arrs.size()>=10 && arrs[9]){
|
||||
o.put("way", arrs[9])
|
||||
}
|
||||
rets << o
|
||||
}catch(Exception e){
|
||||
println arrs
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
index++
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def addTownByTownShape(){
|
||||
def i = 0
|
||||
def headers = ["_id","province","city","district","lng","lat","data_source","乡镇街道", "新区域", "方式"]
|
||||
def headers = ["_id","province","city","district","lng","lat","data_source","乡镇街道", "新区域", "方式", "标准乡镇"]
|
||||
while(i<50){
|
||||
def fileName = "/home/goojia/gj/new_poi/input/no_township_${i}.csv"
|
||||
def fileName = "/home/goojia/gj/new_poi/input/map/no_township_${i}.csv"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\数据建设\\乡镇街道\\待补充乡镇街道\\原始\\no_township_${i}.csv"
|
||||
}
|
||||
@@ -398,7 +434,7 @@ class TownStandardBatch extends BaseBatch{
|
||||
def rets = readCSV2(fileName)
|
||||
def rows = addTownshipBatch(rets)
|
||||
if (CommonDatautils.isWindow()) {
|
||||
com.fangjia.script.gj.ccb.FileUtils.batchCreateFileByCustomSuffix(rows, "D:\\数据\\数据建设\\乡镇街道\\待补充乡镇街道\\结果\\no_township_result_${i}.csv", headers, "utf-8", false, ',', '');
|
||||
com.fangjia.script.gj.ccb.FileUtils.batchCreateFileByCustomSuffix(rows, "D:\\数据\\数据建设\\乡镇街道\\待补充乡镇街道\\结果-地图\\no_township_result_${i}.csv", headers, "utf-8", false, ',', '');
|
||||
}else{
|
||||
com.fangjia.script.gj.ccb.FileUtils.batchCreateFileByCustomSuffix(rows, "/home/goojia/gj/new_poi/output/no_township_result_${i}.csv", headers, "utf-8", false, ',', '');
|
||||
}
|
||||
@@ -425,10 +461,10 @@ class TownStandardBatch extends BaseBatch{
|
||||
def townMap = CommonDatautils.queryTownByCity(r.city, ["city", "district", "mzb_district_name", "street", "alias", "shape"])
|
||||
def set = rMap.keySet()
|
||||
println totalCount.get() +" "+r.city + " " + set.size()
|
||||
if (set.size() >= 1) {
|
||||
// if (set.size() >= 1) {
|
||||
addTown(r, townMap, ["lon":r.lng.toString().toDouble(), "lat":r.lat.toString().toDouble()])
|
||||
results << [r.id, r.province, r.city, r.district, r.lng, r.lat, r.data_source, r.township, r.new_district, r.town_source]
|
||||
}
|
||||
results << [r.id, r.province, r.city, r.district, r.lng, r.lat, r.data_source, r.township, r.new_district, r.town_source, r.strand_town]
|
||||
// }
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
@@ -444,52 +480,87 @@ class TownStandardBatch extends BaseBatch{
|
||||
|
||||
def addTown(saveObj, townMap, location) {
|
||||
//在poi本区域下通过乡镇街道围栏选出所在乡镇街道
|
||||
def towns = townMap.get(saveObj.district)
|
||||
if (towns) {
|
||||
for (def town : towns) {
|
||||
if (town.shape && town.shape.coordinates) {
|
||||
def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates))
|
||||
if (flag) {
|
||||
saveObj.township = town.street
|
||||
saveObj.town_source = "同区域"
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// def towns = townMap.get(saveObj.district)
|
||||
// if (towns) {
|
||||
// for (def town : towns) {
|
||||
// if (town.shape && town.shape.coordinates) {
|
||||
// def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates))
|
||||
// if (flag) {
|
||||
// saveObj.township = town.street
|
||||
// saveObj.town_source = "同区域"
|
||||
// saveObj.strand_town = "是"
|
||||
// break
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//忽略poi中的区域 通过乡镇街道围栏选出所在乡镇街道
|
||||
// if(!saveObj.township){
|
||||
// def townNames = townMap.keySet()
|
||||
// for(def townName : townNames){
|
||||
// towns = townMap.get(townName)
|
||||
// for (def town : towns) {
|
||||
// if (town.shape && town.shape.coordinates) {
|
||||
// def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates))
|
||||
// if (flag) {
|
||||
// saveObj.township = town.street
|
||||
// saveObj.new_district = town.mzb_district_name
|
||||
// saveObj.town_source = "忽略区域"
|
||||
// saveObj.strand_town = isStandTown(townMap, saveObj.new_district, saveObj.township)
|
||||
// break
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// if(saveObj.township){
|
||||
// break;
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//通过调取地图api接口获取乡镇街道
|
||||
if(!saveObj.township){
|
||||
def townNames = townMap.keySet()
|
||||
for(def townName : townNames){
|
||||
towns = townMap.get(townName)
|
||||
for (def town : towns) {
|
||||
if (town.shape && town.shape.coordinates) {
|
||||
def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates))
|
||||
if (flag) {
|
||||
saveObj.township = town.street
|
||||
saveObj.new_district = town.mzb_district_name
|
||||
saveObj.town_source = "忽略区域"
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if(saveObj.township){
|
||||
break;
|
||||
def gaodeMap = getTownshipInfo(location.lon+","+location.lat)
|
||||
if(gaodeMap && gaodeMap.township){
|
||||
saveObj.township = gaodeMap.township
|
||||
saveObj.new_district = gaodeMap.district
|
||||
saveObj.town_source = "高德API"
|
||||
if(townMap.size()>=1){
|
||||
saveObj.strand_town = isStandTown(townMap, saveObj.new_district, saveObj.township)
|
||||
}
|
||||
}
|
||||
}
|
||||
//通过调取地图api接口获取乡镇街道
|
||||
|
||||
// //通过调取地图api接口获取乡镇街道
|
||||
// if(!saveObj.township){
|
||||
// def gaodeMap = getTownshipInfo(location.lon+","+location.lat)
|
||||
// if(gaodeMap && gaodeMap.township){
|
||||
// saveObj.township = gaodeMap.township
|
||||
// saveObj.new_district = gaodeMap.district
|
||||
// saveObj.town_source = "高德API"
|
||||
// def gaodeMap = BaiduMapUtils.geocoding(location.lat+","+location.lon)
|
||||
// if(gaodeMap && gaodeMap.addressComponent) {
|
||||
// def addressComponent = gaodeMap.addressComponent
|
||||
// if(addressComponent){
|
||||
// if(addressComponent.town){
|
||||
// saveObj.township = addressComponent.town
|
||||
// saveObj.new_district = addressComponent.district
|
||||
// saveObj.town_source = "百度API"
|
||||
// saveObj.strand_town = isStandTown(townMap, saveObj.new_district, saveObj.township)
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
return saveObj
|
||||
}
|
||||
|
||||
def isStandTown(townMap, district, town){
|
||||
def towns = townMap.get(district)
|
||||
def flag = "非"
|
||||
if(towns){
|
||||
for(def t : towns){
|
||||
if(t.street == town){
|
||||
flag = "是"
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return flag
|
||||
}
|
||||
|
||||
|
||||
def getTownshipInfo(def glocation){
|
||||
def township = null
|
||||
@@ -498,9 +569,11 @@ class TownStandardBatch extends BaseBatch{
|
||||
def format_addr = null
|
||||
def stop = false
|
||||
def count = 0
|
||||
while(!stop){
|
||||
def jsonObject = null
|
||||
// while(!stop){
|
||||
if(glocation){
|
||||
def jsonObject = GaoDeApiUtils.geocode(glocation)
|
||||
jsonObject = GaoDeApiUtils.geocode(glocation)
|
||||
|
||||
if(jsonObject && jsonObject.regeocode) {
|
||||
def addressComponent = jsonObject.regeocode.addressComponent
|
||||
if(addressComponent){
|
||||
@@ -512,14 +585,15 @@ class TownStandardBatch extends BaseBatch{
|
||||
}
|
||||
}
|
||||
}
|
||||
if(district || count>10){
|
||||
stop = true
|
||||
}else{
|
||||
count ++
|
||||
Thread.sleep(3000)
|
||||
print "."
|
||||
}
|
||||
}
|
||||
// if(jsonObject.status==0 || count>3){
|
||||
// stop = true
|
||||
// println "高德调用超3次"
|
||||
// }else{
|
||||
// count ++
|
||||
// Thread.sleep(3000)
|
||||
// print "."
|
||||
// }
|
||||
// }
|
||||
|
||||
def map = [:]
|
||||
map.city = city
|
||||
@@ -529,6 +603,72 @@ class TownStandardBatch extends BaseBatch{
|
||||
return map;
|
||||
}
|
||||
|
||||
def updateTownData(){
|
||||
def i = 0
|
||||
while(i<50) {
|
||||
def fileName = "/home/goojia/gj/new_poi/output/no_township_result_${i}.csv"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\数据建设\\乡镇街道\\待补充乡镇街道\\结果\\no_township_result_${i}.csv"
|
||||
}
|
||||
File file = new File(fileName);
|
||||
if (file.exists()) {
|
||||
def rets = readCSV3(fileName)
|
||||
println file.getAbsoluteFile().getName() +" "+(rets.size()*100/200000)+"%"+" "+rets.size()
|
||||
// if(rets){
|
||||
// updateToEs(rets)
|
||||
// }
|
||||
|
||||
}
|
||||
i++
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def updateToEs(rows){
|
||||
BulkRequest request = new BulkRequest();
|
||||
int batchSize = 1;
|
||||
int processedCount = 0;
|
||||
def client = ElasticUtils.getClient89("192.168.0.89")
|
||||
for (int i = 0; i < rows.size(); i++) {
|
||||
def map = rows[i]
|
||||
if(map.containsKey("way")){
|
||||
map.remove("way")
|
||||
}
|
||||
request.add(new IndexRequest("map_poi_2024", "_doc", map.id).source(map, XContentType.JSON));
|
||||
if(i%1000==0){
|
||||
print "."
|
||||
}
|
||||
|
||||
// 批量提交数据
|
||||
if (request.numberOfActions() >= batchSize) {
|
||||
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
// 处理错误
|
||||
print "Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||
} else {
|
||||
processedCount += request.numberOfActions();
|
||||
println "Processed " + processedCount + " documents"
|
||||
}
|
||||
|
||||
request = new BulkRequest();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if (request.numberOfActions() > 0) {
|
||||
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
// 处理错误
|
||||
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||
} else {
|
||||
processedCount += request.numberOfActions();
|
||||
println "Processed " + processedCount + " documents"
|
||||
}
|
||||
}
|
||||
|
||||
println "结束时间 "+new Date()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+33
-9
@@ -9,13 +9,8 @@ import net.sf.json.JSONObject
|
||||
|
||||
class BaiduMapUtils {
|
||||
|
||||
private static String AK = "9E99D3540e4dafb7894f7b9fe27fa6b5";
|
||||
def static AKS = ["hj5KqPjaGA3uivtg8uS8XfAnj9X1kPMr","xjFg7DfDqvQleIZasM3ph8Zi2Lw8kgCg",
|
||||
"4pb6fzrWIzvibDu8L1o8DGoY2h1wRIUq","ZzX6mRFfE5GKn4cpUp24Q3OyUIHnfQAR","79ac4ea49e7dedbf80b0ecd475d07784",
|
||||
"9E99D3540e4dafb7894f7b9fe27fa6b5","W3M5uHfcrmlAdhQjf6v93LCI", "C8bb035aacf602551e829e4c64575224",
|
||||
"B42b12d01f2dee712fd66c3e886158a4", "1a3cde429f38434f1811a75e1a90310c","W3M5uHfcrmlAdhQjf6v93LCI",
|
||||
"C8bb035aacf602551e829e4c64575224", "B42b12d01f2dee712fd66c3e886158a4",
|
||||
"1a3cde429f38434f1811a75e1a90310c","9E99D3540e4dafb7894f7b9fe27fa6b5"];
|
||||
private static String AK = "1a3cde429f38434f1811a75e1a90310c";
|
||||
def static AKS = ["1a3cde429f38434f1811a75e1a90310c","1a3cde429f38434f1811a75e1a90310c"];
|
||||
public static boolean HAS_AVAILABLE_AK = true;
|
||||
|
||||
static def getAddress(lng, lat) throws Exception{
|
||||
@@ -83,8 +78,8 @@ class BaiduMapUtils {
|
||||
throw new RuntimeException("无可用百度AK");
|
||||
}
|
||||
System.out.println("更换AK");
|
||||
for(int i = 0 ;i < AKS.length; i++){
|
||||
if(i == AKS.length - 1){
|
||||
for(int i = 0 ;i < AKS.size(); i++){
|
||||
if(i == AKS.size() - 1){
|
||||
AK = AKS[0];
|
||||
HAS_AVAILABLE_AK = false;
|
||||
throw new RuntimeException("无可用百度AK");
|
||||
@@ -127,6 +122,35 @@ class BaiduMapUtils {
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
public static JSONObject geocoding(locationstr) {
|
||||
String url = "https://api.map.baidu.com/reverse_geocoding/v3/?ak="+AK+"&output=json&coordtype=gcj02ll&location="+locationstr
|
||||
boolean stop = false;
|
||||
int recount = 0;
|
||||
JSONObject jsonObj = null
|
||||
while(!stop && recount<6) {
|
||||
try{
|
||||
HttpRequest request = new HttpRequest();
|
||||
try{
|
||||
String result = request.get(url,null,"UTF-8");
|
||||
def json = JSONObject.fromObject(result);
|
||||
if(json.getInt("status") >= 300){
|
||||
changeAK();
|
||||
recount++
|
||||
}else{
|
||||
jsonObj = json.result
|
||||
}
|
||||
}catch (Exception e){
|
||||
e.printStackTrace();
|
||||
}
|
||||
}catch (Exception e){
|
||||
recount++;
|
||||
println e.getMessage()
|
||||
}
|
||||
}
|
||||
return jsonObj
|
||||
}
|
||||
|
||||
public static def searchByBounds(key, min_lat, min_lng, max_lat, max_lng){
|
||||
boolean stop = false;
|
||||
int recount = 0;
|
||||
|
||||
+4
-1
@@ -12,7 +12,7 @@ import org.jsoup.Jsoup
|
||||
import java.util.zip.GZIPInputStream
|
||||
|
||||
class GaoDeApiUtils {
|
||||
def static GAO_DE_AK = "dfc2245bac8a7b2d5b62e060e59d4eca" //宝剑:c6fcd4bfbc40d73ee762b8115dca3390
|
||||
def static GAO_DE_AK = "94c74c60e102972a8d17b7b75d7403be" //宝剑:c6fcd4bfbc40d73ee762b8115dca3390
|
||||
//15021384995:dfc2245bac8a7b2d5b62e060e59d4eca
|
||||
|
||||
/**
|
||||
@@ -35,6 +35,9 @@ class GaoDeApiUtils {
|
||||
content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8")
|
||||
}
|
||||
JSONObject jsonObj = JSONObject.fromObject(content)
|
||||
if(jsonObj.status!=1){
|
||||
println jsonObj
|
||||
}
|
||||
return jsonObj
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user