diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy index 41588517e..0034cdb20 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy @@ -4,11 +4,16 @@ import com.fangjia.script.BaseBatch import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.xiamen.ElasticUtils89 import com.fangjia.script.looplineinfo.InsidePolygon +import com.fangjia.script.util.BaiduMapUtils +import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.mongodb.BasicDBObject import org.apache.commons.io.FileUtils import org.elasticsearch.action.bulk.BulkRequest +import org.elasticsearch.action.bulk.BulkResponse +import org.elasticsearch.action.index.IndexRequest +import org.elasticsearch.action.support.WriteRequest import org.elasticsearch.action.update.UpdateRequest import org.elasticsearch.client.RequestOptions import org.elasticsearch.common.xcontent.XContentType @@ -31,7 +36,7 @@ class TownStandardBatch extends BaseBatch{ def fullCityMap = CommonDatautils.queryFullNameCity() def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity() def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity() - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(100) def updateList = [] def addAliasList = [] @@ -49,7 +54,8 @@ class TownStandardBatch extends BaseBatch{ def TownStandard(){ // townStandardByCat("office") // townStandardByPoi() - addTownByTownShape() + addTownByTownShape()//为没有乡镇街道的poi,补充乡镇街道,忽略区域,并使用地图 +// updateTownData() } def townStandardByCat(def cat){ //"三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德", @@ -385,20 +391,51 @@ class TownStandardBatch extends BaseBatch{ return rets; } + def readCSV3(filename){ + def lines = FileUtils.readLines(new File(filename)) + def rets = []; + def index = 0 + for(line in lines) { + if(line && index>=1){ + String[] arrs = line.split(",") + if(arrs.size()>=10 && arrs[7] && arrs[9]=="同区域"){ + def o = new BasicDBObject() + try{ + o.put("id", arrs[0]) + o.put("township", arrs[7]) + if(arrs.size()>=9 && arrs[8]){ + o.put("district", arrs[8]) + } + if(arrs.size()>=10 && arrs[9]){ + o.put("way", arrs[9]) + } + rets << o + }catch(Exception e){ + println arrs + } + } + + } + index++ + } + return rets; + } + def addTownByTownShape(){ def i = 0 - def headers = ["_id","province","city","district","lng","lat","data_source","乡镇街道", "新区域", "方式"] + def headers = ["_id","province","city","district","lng","lat","data_source","乡镇街道", "新区域", "方式", "标准乡镇"] while(i<50){ - def fileName = "/home/goojia/gj/new_poi/input/no_township_${i}.csv" + def fileName = "/home/goojia/gj/new_poi/input/map/no_township_${i}.csv" if (CommonDatautils.isWindow()) { fileName = "D:\\数据\\数据建设\\乡镇街道\\待补充乡镇街道\\原始\\no_township_${i}.csv" } File file=new File(fileName); if(file.exists()) { def rets = readCSV2(fileName) + println i +" "+ rets.size() def rows = addTownshipBatch(rets) if (CommonDatautils.isWindow()) { - com.fangjia.script.gj.ccb.FileUtils.batchCreateFileByCustomSuffix(rows, "D:\\数据\\数据建设\\乡镇街道\\待补充乡镇街道\\结果\\no_township_result_${i}.csv", headers, "utf-8", false, ',', ''); + com.fangjia.script.gj.ccb.FileUtils.batchCreateFileByCustomSuffix(rows, "D:\\数据\\数据建设\\乡镇街道\\待补充乡镇街道\\结果-地图\\no_township_result_${i}.csv", headers, "utf-8", false, ',', ''); }else{ com.fangjia.script.gj.ccb.FileUtils.batchCreateFileByCustomSuffix(rows, "/home/goojia/gj/new_poi/output/no_township_result_${i}.csv", headers, "utf-8", false, ',', ''); } @@ -425,10 +462,10 @@ class TownStandardBatch extends BaseBatch{ def townMap = CommonDatautils.queryTownByCity(r.city, ["city", "district", "mzb_district_name", "street", "alias", "shape"]) def set = rMap.keySet() println totalCount.get() +" "+r.city + " " + set.size() - if (set.size() >= 1) { +// if (set.size() >= 1) { addTown(r, townMap, ["lon":r.lng.toString().toDouble(), "lat":r.lat.toString().toDouble()]) - results << [r.id, r.province, r.city, r.district, r.lng, r.lat, r.data_source, r.township, r.new_district, r.town_source] - } + results << [r.id, r.province, r.city, r.district, r.lng, r.lat, r.data_source, r.township, r.new_district, r.town_source, r.strand_town] +// } } } catch (Exception e) { e.printStackTrace() @@ -444,52 +481,87 @@ class TownStandardBatch extends BaseBatch{ def addTown(saveObj, townMap, location) { //在poi本区域下通过乡镇街道围栏选出所在乡镇街道 - def towns = townMap.get(saveObj.district) - if (towns) { - for (def town : towns) { - if (town.shape && town.shape.coordinates) { - def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates)) - if (flag) { - saveObj.township = town.street - saveObj.town_source = "同区域" - break - } - } - } - } +// def towns = townMap.get(saveObj.district) +// if (towns) { +// for (def town : towns) { +// if (town.shape && town.shape.coordinates) { +// def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates)) +// if (flag) { +// saveObj.township = town.street +// saveObj.town_source = "同区域" +// saveObj.strand_town = "是" +// break +// } +// } +// } +// } //忽略poi中的区域 通过乡镇街道围栏选出所在乡镇街道 +// if(!saveObj.township){ +// def townNames = townMap.keySet() +// for(def townName : townNames){ +// towns = townMap.get(townName) +// for (def town : towns) { +// if (town.shape && town.shape.coordinates) { +// def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates)) +// if (flag) { +// saveObj.township = town.street +// saveObj.new_district = town.mzb_district_name +// saveObj.town_source = "忽略区域" +// saveObj.strand_town = isStandTown(townMap, saveObj.new_district, saveObj.township) +// break +// } +// } +// } +// if(saveObj.township){ +// break; +// } +// } +// } + //通过调取地图api接口获取乡镇街道 if(!saveObj.township){ - def townNames = townMap.keySet() - for(def townName : townNames){ - towns = townMap.get(townName) - for (def town : towns) { - if (town.shape && town.shape.coordinates) { - def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates)) - if (flag) { - saveObj.township = town.street - saveObj.new_district = town.mzb_district_name - saveObj.town_source = "忽略区域" - break - } - } - } - if(saveObj.township){ - break; + def gaodeMap = getTownshipInfo(location.lon+","+location.lat) + if(gaodeMap && gaodeMap.township){ + saveObj.township = gaodeMap.township + saveObj.new_district = gaodeMap.district + saveObj.town_source = "高德API" + if(townMap.size()>=1){ + saveObj.strand_town = isStandTown(townMap, saveObj.new_district, saveObj.township) } } } - //通过调取地图api接口获取乡镇街道 + +// //通过调取地图api接口获取乡镇街道 // if(!saveObj.township){ -// def gaodeMap = getTownshipInfo(location.lon+","+location.lat) -// if(gaodeMap && gaodeMap.township){ -// saveObj.township = gaodeMap.township -// saveObj.new_district = gaodeMap.district -// saveObj.town_source = "高德API" +// def gaodeMap = BaiduMapUtils.geocoding(location.lat+","+location.lon) +// if(gaodeMap && gaodeMap.addressComponent) { +// def addressComponent = gaodeMap.addressComponent +// if(addressComponent){ +// if(addressComponent.town){ +// saveObj.township = addressComponent.town +// saveObj.new_district = addressComponent.district +// saveObj.town_source = "百度API" +// saveObj.strand_town = isStandTown(townMap, saveObj.new_district, saveObj.township) +// } +// } // } // } return saveObj } + def isStandTown(townMap, district, town){ + def towns = townMap.get(district) + def flag = "非" + if(towns){ + for(def t : towns){ + if(t.street == town){ + flag = "是" + break + } + } + } + return flag + } + def getTownshipInfo(def glocation){ def township = null @@ -498,9 +570,11 @@ class TownStandardBatch extends BaseBatch{ def format_addr = null def stop = false def count = 0 - while(!stop){ + def jsonObject = null +// while(!stop){ if(glocation){ - def jsonObject = GaoDeApiUtils.geocode(glocation) + jsonObject = GaoDeApiUtils.geocode(glocation) + if(jsonObject && jsonObject.regeocode) { def addressComponent = jsonObject.regeocode.addressComponent if(addressComponent){ @@ -512,14 +586,15 @@ class TownStandardBatch extends BaseBatch{ } } } - if(district || count>10){ - stop = true - }else{ - count ++ - Thread.sleep(3000) - print "." - } - } +// if(jsonObject.status==0 || count>3){ +// stop = true +// println "高德调用超3次" +// }else{ +// count ++ +// Thread.sleep(3000) +// print "." +// } +// } def map = [:] map.city = city @@ -529,6 +604,72 @@ class TownStandardBatch extends BaseBatch{ return map; } + def updateTownData(){ + def i = 0 + while(i<50) { + def fileName = "/home/goojia/gj/new_poi/output/no_township_result_${i}.csv" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\数据建设\\乡镇街道\\待补充乡镇街道\\结果\\no_township_result_${i}.csv" + } + File file = new File(fileName); + if (file.exists()) { + def rets = readCSV3(fileName) + println file.getAbsoluteFile().getName() +" "+(rets.size()*100/200000)+"%"+" "+rets.size() +// if(rets){ +// updateToEs(rets) +// } + } + i++ + } + } + + + def updateToEs(rows){ + BulkRequest request = new BulkRequest(); + int batchSize = 1; + int processedCount = 0; + def client = ElasticUtils.getClient89("192.168.0.89") + for (int i = 0; i < rows.size(); i++) { + def map = rows[i] + if(map.containsKey("way")){ + map.remove("way") + } + request.add(new IndexRequest("map_poi_2024", "_doc", map.id).source(map, XContentType.JSON)); + if(i%1000==0){ + print "." + } + + // 批量提交数据 + if (request.numberOfActions() >= batchSize) { + request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); + if (bulkResponse.hasFailures()) { + // 处理错误 + print "Bulk request has failures: " + bulkResponse.buildFailureMessage() + } else { + processedCount += request.numberOfActions(); + println "Processed " + processedCount + " documents" + } + + request = new BulkRequest(); + } + + } + + if (request.numberOfActions() > 0) { + request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); + if (bulkResponse.hasFailures()) { + // 处理错误 + println "Bulk request has failures: " + bulkResponse.buildFailureMessage() + } else { + processedCount += request.numberOfActions(); + println "Processed " + processedCount + " documents" + } + } + + println "结束时间 "+new Date() + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InPolygon.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InPolygon.groovy new file mode 100644 index 000000000..4d79373ca --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InPolygon.groovy @@ -0,0 +1,132 @@ +package com.fangjia.script.looplineinfo + +import com.fangjia.script.common.DataHandleUtils + +import java.awt.geom.Point2D; + +class InPolygon { + + + public static List> createPoint(def pointList){ + List> alllist = new ArrayList<>(); + for(def pointArr : pointList){ + List list = new ArrayList<>(); + for(def arr : pointArr){ + double x = DataHandleUtils.getDoubleRound(6, arr[0]); + double y = DataHandleUtils.getDoubleRound(6, arr[1]); + Point2D.Double point = new Point2D.Double(x,y); + list.add(point); + } + alllist << list + } + return alllist + } + + public static def isInArea(Point2D.Double point, List> pointList){ + def flag = false; + for(List pointArr : pointList){ + flag = isPtInPoly(point, pointArr) + if(flag){ + break; + } + } + return flag; + } + + /** + * 判断点是否在多边形内 + *

+ * 整个算法的思路为:作点平行于y轴的射线,这样就可以直接比较交点是否大于点(x,y)中y的值,可以简化判断。 + * + * @param point 检测点 + * @param pts 多边形的顶点 + * @return 点在多边形内返回true, 否则返回false + */ + public static boolean isPtInPoly(Point2D.Double point, List pts) { + int polygonSidesCount = pts.size(); + //如果点位于多边形的顶点或边上,也算做点在多边形内,直接返回true + boolean boundOrVertex = true; + // 射线与多边形相交的次数 + int intersectCount = 0; + //浮点类型计算时候与0比较时候的容差 + double precision = 2e-10; + // 相邻两点 + Point2D.Double p1, p2; + // 当前点 + Point2D.Double p = point; + // 线段起点 + p1 = pts.get(0); + for (int i = 1; i <= polygonSidesCount; ++i) { + //按照点录入的顺序依次检查相邻两点组成的线段和当前的点的关系。 + if (p.equals(p1)) { + //如果当前点就是多边形的顶点之一,直接返回。 + return boundOrVertex; + } + // 线段终点 + p2 = pts.get(i % polygonSidesCount); + if (p.x < Math.min(p1.x, p2.x) || p.x > Math.max(p1.x, p2.x)) { + //点在x轴上的映射,明显超出了线段在x轴上的投影。此时冲着上做射线,肯定没有交点,直接跳过。 + p1 = p2; + continue; + } + if (p.x > Math.min(p1.x, p2.x) && p.x < Math.max(p1.x, p2.x)) { + // 当前点在线段于x轴上的投影内 + if (p.y <= Math.max(p1.y, p2.y)) { + //当前点的 y 坐标小于 线段对y轴投影的最大值 + if (p1.x == p2.x && p.y >= Math.min(p1.y, p2.y)) { + // 若线段同样是平行于y轴,则可以断定,当前点,在多边形的这条垂直于x轴的边上。 + return boundOrVertex; + } + if (p1.y == p2.y) { + // 若线段为平行于x轴的水平线 + if (p1.y == p.y) { + //当前点正好位于该水平线上,直接返回该点位于多边形的一条边上。 + return boundOrVertex; + } else { + //如果当前点在水平线以下,增加一个交点。 + ++intersectCount; + } + } else { + // 如果不是水平线,则用两点式求当前点的x带入多边形线段的直线方程后,对应的y的坐标 + double xInSideLineFormulaResultY = (p.x - p1.x) * (p2.y - p1.y) / (p2.x - p1.x) + p1.y; + if (Math.abs(p.y - xInSideLineFormulaResultY) < precision) { + // 误差允许范围内,改点就在线段上,则表明,该点位于多边形的一个边上。 + return boundOrVertex; + } + + if (p.y < xInSideLineFormulaResultY) { + // 如果线段上取得的y比当前点的y要大,当前做向上的射线,肯定交于上方的一个点。 + ++intersectCount; + } + } + } + } else { + // 当前点不在线段投影到x轴的区间中 + if (p.x == p2.x && p.y <= p2.y) { + // 但恰好位于线段终点的x坐标对应的平行于y轴上的线上的低于终点y的一点 + // 此时检查下一点能否将其x边界包含。 + Point2D.Double p3 = pts.get((i + 1) % polygonSidesCount); + if (p.x >= Math.min(p1.x, p3.x) && p.x <= Math.max(p1.x, p3.x)) { + // 若当前点的x坐标位于 p1和p3组成的线段关于x轴的投影中,则记为该点的射线只穿过端点一次。 + ++intersectCount; + } else { + // 若当前点的x坐标不能包含在p1和p3组成的线段关于x轴的投影中,则点射线通过的两条线段组成了一个弯折的部分, + // 此时我们记射线穿过该端点两次 + intersectCount += 2; + } + // 此判断的核心思路是由点在两条线段的内部还是外部去思考得出的 + } + } + // 进行下一个线段的判断 + p1 = p2; + } + if (intersectCount % 2 == 0) { + //偶数在多边形外 + return false; + } else { + //奇数在多边形内 + return true; + } + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy index 58a13a34d..25f5c45ed 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy @@ -324,4 +324,8 @@ class InsidePolygon { } return name; } + + + + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/BaiduMapUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/BaiduMapUtils.groovy index cb84af2c7..9a4003b7d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/BaiduMapUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/BaiduMapUtils.groovy @@ -9,13 +9,8 @@ import net.sf.json.JSONObject class BaiduMapUtils { - private static String AK = "9E99D3540e4dafb7894f7b9fe27fa6b5"; - def static AKS = ["hj5KqPjaGA3uivtg8uS8XfAnj9X1kPMr","xjFg7DfDqvQleIZasM3ph8Zi2Lw8kgCg", - "4pb6fzrWIzvibDu8L1o8DGoY2h1wRIUq","ZzX6mRFfE5GKn4cpUp24Q3OyUIHnfQAR","79ac4ea49e7dedbf80b0ecd475d07784", - "9E99D3540e4dafb7894f7b9fe27fa6b5","W3M5uHfcrmlAdhQjf6v93LCI", "C8bb035aacf602551e829e4c64575224", - "B42b12d01f2dee712fd66c3e886158a4", "1a3cde429f38434f1811a75e1a90310c","W3M5uHfcrmlAdhQjf6v93LCI", - "C8bb035aacf602551e829e4c64575224", "B42b12d01f2dee712fd66c3e886158a4", - "1a3cde429f38434f1811a75e1a90310c","9E99D3540e4dafb7894f7b9fe27fa6b5"]; + private static String AK = "1a3cde429f38434f1811a75e1a90310c"; + def static AKS = ["1a3cde429f38434f1811a75e1a90310c","1a3cde429f38434f1811a75e1a90310c"]; public static boolean HAS_AVAILABLE_AK = true; static def getAddress(lng, lat) throws Exception{ @@ -83,8 +78,8 @@ class BaiduMapUtils { throw new RuntimeException("无可用百度AK"); } System.out.println("更换AK"); - for(int i = 0 ;i < AKS.length; i++){ - if(i == AKS.length - 1){ + for(int i = 0 ;i < AKS.size(); i++){ + if(i == AKS.size() - 1){ AK = AKS[0]; HAS_AVAILABLE_AK = false; throw new RuntimeException("无可用百度AK"); @@ -127,6 +122,35 @@ class BaiduMapUtils { return null } + + public static JSONObject geocoding(locationstr) { + String url = "https://api.map.baidu.com/reverse_geocoding/v3/?ak="+AK+"&output=json&coordtype=gcj02ll&location="+locationstr + boolean stop = false; + int recount = 0; + JSONObject jsonObj = null + while(!stop && recount<6) { + try{ + HttpRequest request = new HttpRequest(); + try{ + String result = request.get(url,null,"UTF-8"); + def json = JSONObject.fromObject(result); + if(json.getInt("status") >= 300){ + changeAK(); + recount++ + }else{ + jsonObj = json.result + } + }catch (Exception e){ + e.printStackTrace(); + } + }catch (Exception e){ + recount++; + println e.getMessage() + } + } + return jsonObj + } + public static def searchByBounds(key, min_lat, min_lng, max_lat, max_lng){ boolean stop = false; int recount = 0; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy index 7c41bcc1c..f4801828f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy @@ -12,7 +12,7 @@ import org.jsoup.Jsoup import java.util.zip.GZIPInputStream class GaoDeApiUtils { - def static GAO_DE_AK = "dfc2245bac8a7b2d5b62e060e59d4eca" //宝剑:c6fcd4bfbc40d73ee762b8115dca3390 + def static GAO_DE_AK = "94c74c60e102972a8d17b7b75d7403be" //宝剑:c6fcd4bfbc40d73ee762b8115dca3390 //15021384995:dfc2245bac8a7b2d5b62e060e59d4eca /** @@ -35,6 +35,9 @@ class GaoDeApiUtils { content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8") } JSONObject jsonObj = JSONObject.fromObject(content) + if(jsonObj.status!="1"){ + println jsonObj + } return jsonObj }