Merge remote-tracking branch 'origin/master'

This commit is contained in:
majintao
2025-02-24 16:22:45 +08:00
5 changed files with 396 additions and 73 deletions
@@ -4,11 +4,16 @@ import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.BaiduMapUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import org.apache.commons.io.FileUtils
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.action.support.WriteRequest
import org.elasticsearch.action.update.UpdateRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.common.xcontent.XContentType
@@ -31,7 +36,7 @@ class TownStandardBatch extends BaseBatch{
def fullCityMap = CommonDatautils.queryFullNameCity()
def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity()
def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity()
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
def updateList = []
def addAliasList = []
@@ -49,7 +54,8 @@ class TownStandardBatch extends BaseBatch{
def TownStandard(){
// townStandardByCat("office")
// townStandardByPoi()
addTownByTownShape()
addTownByTownShape()//为没有乡镇街道的poi,补充乡镇街道,忽略区域,并使用地图
// updateTownData()
}
def townStandardByCat(def cat){
//"三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德",
@@ -374,7 +380,11 @@ class TownStandardBatch extends BaseBatch{
o.put("district", arrs[4])
o.put("lng", arrs[5])
o.put("lat", arrs[6])
o.put("data_source", arrs[7])
if(arrs.size()<=7){
o.put("data_source","")
}else{
o.put("data_source", arrs[7])
}
rets << o
}catch(Exception e){
println arrs
@@ -385,20 +395,51 @@ class TownStandardBatch extends BaseBatch{
return rets;
}
def readCSV3(filename){
def lines = FileUtils.readLines(new File(filename))
def rets = [];
def index = 0
for(line in lines) {
if(line && index>=1){
String[] arrs = line.split(",")
if(arrs.size()>=10 && arrs[7] && arrs[9]=="同区域"){
def o = new BasicDBObject()
try{
o.put("id", arrs[0])
o.put("township", arrs[7])
if(arrs.size()>=9 && arrs[8]){
o.put("district", arrs[8])
}
if(arrs.size()>=10 && arrs[9]){
o.put("way", arrs[9])
}
rets << o
}catch(Exception e){
println arrs
}
}
}
index++
}
return rets;
}
def addTownByTownShape(){
def i = 0
def headers = ["_id","province","city","district","lng","lat","data_source","乡镇街道", "新区域", "方式"]
def headers = ["_id","province","city","district","lng","lat","data_source","乡镇街道", "新区域", "方式", "标准乡镇"]
while(i<50){
def fileName = "/home/goojia/gj/new_poi/input/no_township_${i}.csv"
def fileName = "/home/goojia/gj/new_poi/input/map/no_township_${i}.csv"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\数据建设\\乡镇街道\\待补充乡镇街道\\原始\\no_township_${i}.csv"
}
File file=new File(fileName);
if(file.exists()) {
def rets = readCSV2(fileName)
println i +" "+ rets.size()
def rows = addTownshipBatch(rets)
if (CommonDatautils.isWindow()) {
com.fangjia.script.gj.ccb.FileUtils.batchCreateFileByCustomSuffix(rows, "D:\\数据\\数据建设\\乡镇街道\\待补充乡镇街道\\结果\\no_township_result_${i}.csv", headers, "utf-8", false, ',', '');
com.fangjia.script.gj.ccb.FileUtils.batchCreateFileByCustomSuffix(rows, "D:\\数据\\数据建设\\乡镇街道\\待补充乡镇街道\\结果-地图\\no_township_result_${i}.csv", headers, "utf-8", false, ',', '');
}else{
com.fangjia.script.gj.ccb.FileUtils.batchCreateFileByCustomSuffix(rows, "/home/goojia/gj/new_poi/output/no_township_result_${i}.csv", headers, "utf-8", false, ',', '');
}
@@ -422,13 +463,13 @@ class TownStandardBatch extends BaseBatch{
totalCount.addAndGet(1)
def rMap = regionFullNameMap.get(r.city)
if (rMap) {
def townMap = CommonDatautils.queryTownByCity(r.city, ["city", "district", "mzb_district_name", "street", "alias", "shape"])
def set = rMap.keySet()
println totalCount.get() +" "+r.city + " " + set.size()
if (set.size() >= 1) {
def townMap = [:]//CommonDatautils.queryTownByCity(r.city, ["city", "district", "mzb_district_name", "street", "alias", "shape"])
// def set = rMap.keySet()
println totalCount.get() +" "+r.city + " "
// if (set.size() >= 1) {
addTown(r, townMap, ["lon":r.lng.toString().toDouble(), "lat":r.lat.toString().toDouble()])
results << [r.id, r.province, r.city, r.district, r.lng, r.lat, r.data_source, r.township, r.new_district, r.town_source]
}
results << [r.id, r.province, r.city, r.district, r.lng, r.lat, r.data_source, r.township, r.new_district, r.town_source, r.strand_town]
// }
}
} catch (Exception e) {
e.printStackTrace()
@@ -444,52 +485,88 @@ class TownStandardBatch extends BaseBatch{
def addTown(saveObj, townMap, location) {
//在poi本区域下通过乡镇街道围栏选出所在乡镇街道
def towns = townMap.get(saveObj.district)
if (towns) {
for (def town : towns) {
if (town.shape && town.shape.coordinates) {
def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates))
if (flag) {
saveObj.township = town.street
saveObj.town_source = "同区域"
break
}
}
}
}
// def towns = townMap.get(saveObj.district)
// if (towns) {
// for (def town : towns) {
// if (town.shape && town.shape.coordinates) {
// def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates))
// if (flag) {
// saveObj.township = town.street
// saveObj.town_source = "同区域"
// saveObj.strand_town = "是"
// break
// }
// }
// }
// }
//忽略poi中的区域 通过乡镇街道围栏选出所在乡镇街道
// if(!saveObj.township){
// def townNames = townMap.keySet()
// for(def townName : townNames){
// towns = townMap.get(townName)
// for (def town : towns) {
// if (town.shape && town.shape.coordinates) {
// def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates))
// if (flag) {
// saveObj.township = town.street
// saveObj.new_district = town.mzb_district_name
// saveObj.town_source = "忽略区域"
// saveObj.strand_town = isStandTown(townMap, saveObj.new_district, saveObj.township)
// break
// }
// }
// }
// if(saveObj.township){
// break;
// }
// }
// }
//通过调取地图api接口获取乡镇街道
if(!saveObj.township){
def townNames = townMap.keySet()
for(def townName : townNames){
towns = townMap.get(townName)
for (def town : towns) {
if (town.shape && town.shape.coordinates) {
def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates))
if (flag) {
saveObj.township = town.street
saveObj.new_district = town.mzb_district_name
saveObj.town_source = "忽略区域"
break
}
}
}
if(saveObj.township){
break;
}
def gaodeMap = getTownshipInfo(location.lon+","+location.lat)
if(gaodeMap && gaodeMap.township){
saveObj.township = gaodeMap.township
saveObj.new_district = gaodeMap.district
saveObj.town_source = "高德API"
saveObj.strand_town = ""
// if(townMap.size()>=1){
// saveObj.strand_town = isStandTown(townMap, saveObj.new_district, saveObj.township)
// }
}
}
//通过调取地图api接口获取乡镇街道
// //通过调取地图api接口获取乡镇街道
// if(!saveObj.township){
// def gaodeMap = getTownshipInfo(location.lon+","+location.lat)
// if(gaodeMap && gaodeMap.township){
// saveObj.township = gaodeMap.township
// saveObj.new_district = gaodeMap.district
// saveObj.town_source = "高德API"
// def gaodeMap = BaiduMapUtils.geocoding(location.lat+","+location.lon)
// if(gaodeMap && gaodeMap.addressComponent) {
// def addressComponent = gaodeMap.addressComponent
// if(addressComponent){
// if(addressComponent.town){
// saveObj.township = addressComponent.town
// saveObj.new_district = addressComponent.district
// saveObj.town_source = "百度API"
// saveObj.strand_town = isStandTown(townMap, saveObj.new_district, saveObj.township)
// }
// }
// }
// }
return saveObj
}
def isStandTown(townMap, district, town){
def towns = townMap.get(district)
def flag = "非"
if(towns){
for(def t : towns){
if(t.street == town){
flag = "是"
break
}
}
}
return flag
}
def getTownshipInfo(def glocation){
def township = null
@@ -498,9 +575,11 @@ class TownStandardBatch extends BaseBatch{
def format_addr = null
def stop = false
def count = 0
while(!stop){
def jsonObject = null
// while(!stop){
if(glocation){
def jsonObject = GaoDeApiUtils.geocode(glocation)
jsonObject = GaoDeApiUtils.geocode(glocation)
if(jsonObject && jsonObject.regeocode) {
def addressComponent = jsonObject.regeocode.addressComponent
if(addressComponent){
@@ -512,14 +591,15 @@ class TownStandardBatch extends BaseBatch{
}
}
}
if(district || count>10){
stop = true
}else{
count ++
Thread.sleep(3000)
print "."
}
}
// if(jsonObject.status==0 || count>3){
// stop = true
// println "高德调用超3次"
// }else{
// count ++
// Thread.sleep(3000)
// print "."
// }
// }
def map = [:]
map.city = city
@@ -529,6 +609,72 @@ class TownStandardBatch extends BaseBatch{
return map;
}
def updateTownData(){
def i = 0
while(i<50) {
def fileName = "/home/goojia/gj/new_poi/output/no_township_result_${i}.csv"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\数据建设\\乡镇街道\\待补充乡镇街道\\结果\\no_township_result_${i}.csv"
}
File file = new File(fileName);
if (file.exists()) {
def rets = readCSV3(fileName)
println file.getAbsoluteFile().getName() +" "+(rets.size()*100/200000)+"%"+" "+rets.size()
// if(rets){
// updateToEs(rets)
// }
}
i++
}
}
def updateToEs(rows){
BulkRequest request = new BulkRequest();
int batchSize = 1;
int processedCount = 0;
def client = ElasticUtils.getClient89("192.168.0.89")
for (int i = 0; i < rows.size(); i++) {
def map = rows[i]
if(map.containsKey("way")){
map.remove("way")
}
request.add(new IndexRequest("map_poi_2024", "_doc", map.id).source(map, XContentType.JSON));
if(i%1000==0){
print "."
}
// 批量提交数据
if (request.numberOfActions() >= batchSize) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
print "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents"
}
request = new BulkRequest();
}
}
if (request.numberOfActions() > 0) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents"
}
}
println "结束时间 "+new Date()
}
}
@@ -0,0 +1,132 @@
package com.fangjia.script.looplineinfo
import com.fangjia.script.common.DataHandleUtils
import java.awt.geom.Point2D;
class InPolygon {
public static List<List<Point2D.Double>> createPoint(def pointList){
List<List<Point2D.Double>> alllist = new ArrayList<>();
for(def pointArr : pointList){
List<Point2D.Double> list = new ArrayList<>();
for(def arr : pointArr){
double x = DataHandleUtils.getDoubleRound(6, arr[0]);
double y = DataHandleUtils.getDoubleRound(6, arr[1]);
Point2D.Double point = new Point2D.Double(x,y);
list.add(point);
}
alllist << list
}
return alllist
}
public static def isInArea(Point2D.Double point, List<List<Point2D.Double>> pointList){
def flag = false;
for(List<Point2D.Double> pointArr : pointList){
flag = isPtInPoly(point, pointArr)
if(flag){
break;
}
}
return flag;
}
/**
* 判断点是否在多边形内
* <p>
* 整个算法的思路为:作点平行于y轴的射线,这样就可以直接比较交点是否大于点(x,y)中y的值,可以简化判断。
*
* @param point 检测点
* @param pts 多边形的顶点
* @return 点在多边形内返回true, 否则返回false
*/
public static boolean isPtInPoly(Point2D.Double point, List<Point2D.Double> pts) {
int polygonSidesCount = pts.size();
//如果点位于多边形的顶点或边上,也算做点在多边形内,直接返回true
boolean boundOrVertex = true;
// 射线与多边形相交的次数
int intersectCount = 0;
//浮点类型计算时候与0比较时候的容差
double precision = 2e-10;
// 相邻两点
Point2D.Double p1, p2;
// 当前点
Point2D.Double p = point;
// 线段起点
p1 = pts.get(0);
for (int i = 1; i <= polygonSidesCount; ++i) {
//按照点录入的顺序依次检查相邻两点组成的线段和当前的点的关系。
if (p.equals(p1)) {
//如果当前点就是多边形的顶点之一,直接返回。
return boundOrVertex;
}
// 线段终点
p2 = pts.get(i % polygonSidesCount);
if (p.x < Math.min(p1.x, p2.x) || p.x > Math.max(p1.x, p2.x)) {
//点在x轴上的映射,明显超出了线段在x轴上的投影。此时冲着上做射线,肯定没有交点,直接跳过。
p1 = p2;
continue;
}
if (p.x > Math.min(p1.x, p2.x) && p.x < Math.max(p1.x, p2.x)) {
// 当前点在线段于x轴上的投影内
if (p.y <= Math.max(p1.y, p2.y)) {
//当前点的 y 坐标小于 线段对y轴投影的最大值
if (p1.x == p2.x && p.y >= Math.min(p1.y, p2.y)) {
// 若线段同样是平行于y轴,则可以断定,当前点,在多边形的这条垂直于x轴的边上。
return boundOrVertex;
}
if (p1.y == p2.y) {
// 若线段为平行于x轴的水平线
if (p1.y == p.y) {
//当前点正好位于该水平线上,直接返回该点位于多边形的一条边上。
return boundOrVertex;
} else {
//如果当前点在水平线以下,增加一个交点。
++intersectCount;
}
} else {
// 如果不是水平线,则用两点式求当前点的x带入多边形线段的直线方程后,对应的y的坐标
double xInSideLineFormulaResultY = (p.x - p1.x) * (p2.y - p1.y) / (p2.x - p1.x) + p1.y;
if (Math.abs(p.y - xInSideLineFormulaResultY) < precision) {
// 误差允许范围内,改点就在线段上,则表明,该点位于多边形的一个边上。
return boundOrVertex;
}
if (p.y < xInSideLineFormulaResultY) {
// 如果线段上取得的y比当前点的y要大,当前做向上的射线,肯定交于上方的一个点。
++intersectCount;
}
}
}
} else {
// 当前点不在线段投影到x轴的区间中
if (p.x == p2.x && p.y <= p2.y) {
// 但恰好位于线段终点的x坐标对应的平行于y轴上的线上的低于终点y的一点
// 此时检查下一点能否将其x边界包含。
Point2D.Double p3 = pts.get((i + 1) % polygonSidesCount);
if (p.x >= Math.min(p1.x, p3.x) && p.x <= Math.max(p1.x, p3.x)) {
// 若当前点的x坐标位于 p1和p3组成的线段关于x轴的投影中,则记为该点的射线只穿过端点一次。
++intersectCount;
} else {
// 若当前点的x坐标不能包含在p1和p3组成的线段关于x轴的投影中,则点射线通过的两条线段组成了一个弯折的部分,
// 此时我们记射线穿过该端点两次
intersectCount += 2;
}
// 此判断的核心思路是由点在两条线段的内部还是外部去思考得出的
}
}
// 进行下一个线段的判断
p1 = p2;
}
if (intersectCount % 2 == 0) {
//偶数在多边形外
return false;
} else {
//奇数在多边形内
return true;
}
}
}
@@ -324,4 +324,8 @@ class InsidePolygon {
}
return name;
}
}
@@ -9,13 +9,8 @@ import net.sf.json.JSONObject
class BaiduMapUtils {
private static String AK = "9E99D3540e4dafb7894f7b9fe27fa6b5";
def static AKS = ["hj5KqPjaGA3uivtg8uS8XfAnj9X1kPMr","xjFg7DfDqvQleIZasM3ph8Zi2Lw8kgCg",
"4pb6fzrWIzvibDu8L1o8DGoY2h1wRIUq","ZzX6mRFfE5GKn4cpUp24Q3OyUIHnfQAR","79ac4ea49e7dedbf80b0ecd475d07784",
"9E99D3540e4dafb7894f7b9fe27fa6b5","W3M5uHfcrmlAdhQjf6v93LCI", "C8bb035aacf602551e829e4c64575224",
"B42b12d01f2dee712fd66c3e886158a4", "1a3cde429f38434f1811a75e1a90310c","W3M5uHfcrmlAdhQjf6v93LCI",
"C8bb035aacf602551e829e4c64575224", "B42b12d01f2dee712fd66c3e886158a4",
"1a3cde429f38434f1811a75e1a90310c","9E99D3540e4dafb7894f7b9fe27fa6b5"];
private static String AK = "1a3cde429f38434f1811a75e1a90310c";
def static AKS = ["1a3cde429f38434f1811a75e1a90310c","1a3cde429f38434f1811a75e1a90310c"];
public static boolean HAS_AVAILABLE_AK = true;
static def getAddress(lng, lat) throws Exception{
@@ -83,8 +78,8 @@ class BaiduMapUtils {
throw new RuntimeException("无可用百度AK");
}
System.out.println("更换AK");
for(int i = 0 ;i < AKS.length; i++){
if(i == AKS.length - 1){
for(int i = 0 ;i < AKS.size(); i++){
if(i == AKS.size() - 1){
AK = AKS[0];
HAS_AVAILABLE_AK = false;
throw new RuntimeException("无可用百度AK");
@@ -127,6 +122,35 @@ class BaiduMapUtils {
return null
}
public static JSONObject geocoding(locationstr) {
String url = "https://api.map.baidu.com/reverse_geocoding/v3/?ak="+AK+"&output=json&coordtype=gcj02ll&location="+locationstr
boolean stop = false;
int recount = 0;
JSONObject jsonObj = null
while(!stop && recount<6) {
try{
HttpRequest request = new HttpRequest();
try{
String result = request.get(url,null,"UTF-8");
def json = JSONObject.fromObject(result);
if(json.getInt("status") >= 300){
changeAK();
recount++
}else{
jsonObj = json.result
}
}catch (Exception e){
e.printStackTrace();
}
}catch (Exception e){
recount++;
println e.getMessage()
}
}
return jsonObj
}
public static def searchByBounds(key, min_lat, min_lng, max_lat, max_lng){
boolean stop = false;
int recount = 0;
@@ -12,7 +12,8 @@ import org.jsoup.Jsoup
import java.util.zip.GZIPInputStream
class GaoDeApiUtils {
def static GAO_DE_AK = "dfc2245bac8a7b2d5b62e060e59d4eca" //宝剑:c6fcd4bfbc40d73ee762b8115dca3390
//94c74c60e102972a8d17b7b75d7403be
def static GAO_DE_AK = "94c74c60e102972a8d17b7b75d7403be" //宝剑:c6fcd4bfbc40d73ee762b8115dca3390
//15021384995:dfc2245bac8a7b2d5b62e060e59d4eca
/**
@@ -29,12 +30,28 @@ class GaoDeApiUtils {
params.put("location", location)
params.put("extensions","all")
String content
try {
content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8")
} catch (Exception e) {
content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8")
def stop = true
JSONObject jsonObj = null
def errorCount = 1
while (stop){
try {
content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8")
} catch (Exception e) {
content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8")
}
jsonObj = JSONObject.fromObject(content)
if(jsonObj.status!="1"){
println jsonObj
errorCount ++
Thread.sleep(3000)
if(errorCount>3){
stop = false
}
}else{
stop = false
}
}
JSONObject jsonObj = JSONObject.fromObject(content)
return jsonObj
}