diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy index e8a43efdb..8c42ab478 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy @@ -1,15 +1,30 @@ package com.fangjia.script.temp.shenjian +import com.fangjia.script.lixue.util.DBUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject + /** * @author shenjian* @since 2021/5/18 */ class Temp { static void main(String[] args) { - double val = 1.0 - if (val == 1.0) { - println(true) + DBObject query = new BasicDBObject() + query.put("city", "重庆") + query.put("region", "北碚") + query.put("cat", "district") + query.put("visible", 0) + query.put("name", "人民村") + DBUtils.seaweed.find(query).limit(1).each { obj -> + println(obj) } - } + +// 万友七季城C区 经度纬度不同106.569161 29.498502 原始106.56881 29.50058 距离231.7003米 +// 马道子 经度纬度不同106.384087 29.66009 原始106.450656 29.550612 距离12306.7728米 +// 富洲新城 经度纬度不同106.474475 29.537152 原始106.420021 29.537655 距离1836.9266米 +// 滨江国际 经度纬度不同109.466428 31.01454 原始109.46799 31.016907 距离308.3405米 +// 澳海澜庭 经度纬度不同109.389067 31.020248 原始109.389069 31.02025 距离0.3029米 +// 富洲新城 经度纬度不同106.474475 29.537152 原始106.420021 29.537655 距离1836.9266米 } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy index 5d676e7c9..3bffc1300 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy @@ -59,11 +59,12 @@ class AroundExportByNewWay { query.put("cat", "district") query.put("visible", 0) query.put("name", district.get(24)) + Set set = new HashSet<>() DBUtils.seaweed.find(query, new BasicDBObject("lng", 1).append("lat", 1)).limit(1).each {obj -> if (obj.lng && obj.lat) { def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lng/ 10000000000d) def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lat / 10000000000d) - calcAroundInfo(lat, lng, result, id) + calcAroundInfo(lat, lng, result, id, set) } } } else { @@ -106,7 +107,7 @@ class AroundExportByNewWay { List currRet = resultMap.get(district_code + name) Collections.sort(currRet) ret.addAll(currRet) - if (ret.size() > 600000) { + if (ret.size() > 700000) { map.put(key++, ret) ret = [] } @@ -123,7 +124,7 @@ class AroundExportByNewWay { executorService.shutdown() } - static calcAroundInfo(def lat, def lng, def result, def id) { + static calcAroundInfo(def lat, def lng, def result, def id, def set) { // 周围半径为3千米 def mapLatLng = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 3000) DBObject q = new BasicDBObject() @@ -136,12 +137,16 @@ class AroundExportByNewWay { if (distance == 0) { distance = 50 } - AroundDTO aroundDTO = new AroundDTO() - aroundDTO.id = id - aroundDTO.name = obj.name - aroundDTO.type = formatType(obj.typecode) - aroundDTO.distance = distance.toInteger() - result.add(aroundDTO) + if (!set.contains(obj.name)) { + AroundDTO aroundDTO = new AroundDTO() + aroundDTO.id = id + aroundDTO.name = obj.name + aroundDTO.type = formatType(obj.typecode) + aroundDTO.distance = distance.toInteger() + result.add(aroundDTO) + + set.add(obj.name) + } } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundFormatExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundFormatExport.groovy index b445e881c..9a3f67a3c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundFormatExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundFormatExport.groovy @@ -1,31 +1,36 @@ package com.fangjia.script.temp.shenjian.cqrcb -import cn.emay.mina.util.ConcurrentHashSet -import com.fangjia.script.coredata.util.CollectionUtils -import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.alibaba.excel.EasyExcel +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.district.Distance +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.temp.shenjian.dto.AroundDTO import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject +import com.mongodb.QueryOperators +import org.apache.commons.collections.map.HashedMap import java.util.concurrent.ConcurrentHashMap -import java.util.concurrent.ConcurrentLinkedQueue import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicInteger /** - * 小区周围信息格式化输出 + * 周边数据一千米配套数量统计 * * @author shenjian - * @since 2021/5/15 + * @since 2021/5/16 */ class AroundFormatExport { - static DBCollection conn = MongoUtils.getCollectionByName("seeddb_dev", "arround_temp", "192.168.0.235", 27018) + static DBCollection poiConn = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017) static def resultMap = new ConcurrentHashMap() + static def unMatchedMap = new ConcurrentHashMap() static ExecutorService executorService = Executors.newFixedThreadPool(30) static void main(String[] args) { @@ -38,124 +43,148 @@ class AroundFormatExport { @Override void run() { try { + def id = district.get(0) def district_code = district.get(2) def name = district.get(3) - def temp = [] - if (district.get(20) != null && district.get(21) != null) { - def matched_city = district.get(22) - def matched_region = district.get(23) - def matched_name = district.get(24) + List result = [] + if (district.size() > 21 && district.get(20) != null && district.get(21) != null) { + def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, district.get(20) / 10000000000d) + def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, district.get(21) / 10000000000d) + calcAroundInfo(lat, lng, result, id) + } else if (district.size() >24 &&district.get(22) != null && district.get(23) != null && district.get(24) != null) { DBObject query = new BasicDBObject() - query.put("matched_city", matched_city) - query.put("matched_region", matched_region) - query.put("matched_name", matched_name) - List ret = new ArrayList() - Map map = new ConcurrentHashMap<>(); - conn.find(query, new BasicDBObject("name": 1, "distance": 1, "matched_region": 1)).batchSize(1000).each { obj -> - dealSpecial(obj, map, ret) + query.put("city", district.get(22)) + query.put("region", district.get(23)) + query.put("cat", "district") + query.put("visible", 0) + query.put("name", district.get(24)) + Set set = new HashSet<>() + DBUtils.seaweed.find(query, new BasicDBObject("lng", 1).append("lat", 1)).limit(1).each {obj -> + if (obj.lng && obj.lat) { + def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lng/ 10000000000d) + def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lat / 10000000000d) + calcAroundInfo(lat, lng, result, id, set) + } } - for (String key : map.keySet()) { - int distance = map.get(key).get("distance") - int count1 = map.get(key).get("count") - ret.add(key + ":" + (distance / count1) + "米") - } - - temp[7] = matched_city - temp[8] = matched_region - temp[9] = matched_name - if (ret.size() > 0) { - def str = ret.join(",") - temp[5] = str - println(str) - } - + } else { + def ret = [] + ret[0] = id + ret[1] = district.get(1) + ret[2] = district_code + ret[3] = name + ret[4] = district.get(4) + unMatchedMap.put(id, ret) } - resultMap.put(district_code + name, temp) + resultMap.put(district_code + name, result) println("当前处理" + count.incrementAndGet() ) } catch (Exception ex) { - + ex.printStackTrace() } finally { latch.countDown() } } }) + }; + latch.await(); + def headArr = ['IDB', '数量'] + List head = new ArrayList<>() + for (String name: headArr) { + List temp = new ArrayList<>() + temp.add(name) + head.add(temp) } - latch.await() + Map map = new HashedMap() + Integer key = 1; def ret = [] - ret << ['IDB','城市名称','区域编码','小区名称','楼栋号','3km范围内地铁站、公交站、银行、医院、商场、学校等名称及距离信息','1km范围内地铁站、公交站、银行、医院、商场、学校等数量'] + AtomicInteger totalCount = new AtomicInteger() for (List district : data) { + def id = district.get(0) def district_code = district.get(2) def name = district.get(3) - ret << resultMap.get(district_code + name) + if (com.fangjia.script.coredata.util.CollectionUtils.isNotEmpty(resultMap.get(district_code + name))) { + totalCount.incrementAndGet() + List currRet = resultMap.get(district_code + name) + ret << [id, currRet.size()] + if (ret.size() > 700000) { + map.put(key++, ret) + ret = [] + } + } else { + ret << [id, 0] + } } - com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套.xlsx", ret, "周边配套"); + map.put(key++, ret) + //com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套.xlsx", ret, "周边配套"); + for (Integer k : map.keySet()) { + //FileUtils.batchCreateFile(map.get(k), "F:\\数据\\重庆农商行\\周边配套"+ k + ".dat", headArr, "utf-8") + EasyExcel.write("F:\\数据\\重庆农商行\\周边配套1千米V1.xlsx").head(head).sheet("周边配套").doWrite(map.get(k)); + } + print(totalCount.get() + " " + (totalCount.get() / data.size())) + printUnMatched(unMatchedMap) executorService.shutdown() } - static dealSpecial(DBObject obj, def map, def ret) { - def name = obj.name - if (name.indexOf("/") != -1) { - return false - } - if (name == "1号楼") { - return false - } - if (name.indexOf("地铁口") == -1 && name.indexOf("公交站") == -1 && name.indexOf("支行") == -1) { - if (name.indexOf("医院") != -1 && !name.endsWith("医院")) { - if (name.indexOf("分院") == -1) { - return false + static calcAroundInfo(def lat, def lng, def result, def id, def set) { + // 周围半径为3千米 + def mapLatLng = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 1000) + DBObject q = new BasicDBObject() + q.put('status', 0) + q.put('lat', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLat).append(QueryOperators.GTE, mapLatLng.minLat)); + q.put('lng', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLng).append(QueryOperators.GTE, mapLatLng.minLng)); + poiConn.find(q, new BasicDBObject("name": 1, "distance": 1, "type": 1, "typecode": 1, "lat": 1, "lng": 1)).batchSize(500).each {obj -> + double distance = Distance.GetDistance(lng, lat, obj.lng, obj.lat) + if (distance <= 1000) { + if (distance == 0) { + distance = 50 + } + if (!set.contains(obj.name)) { + AroundDTO aroundDTO = new AroundDTO() + aroundDTO.id = id + aroundDTO.name = obj.name + aroundDTO.type = formatType(obj.typecode) + aroundDTO.distance = distance.toInteger() + result.add(aroundDTO) + + set.add(obj.name) } } - if (name.indexOf("卫生中心") != -1 && !name.endsWith("卫生中心")) { - return false - } - if (name.indexOf("大学") != -1 && !name.endsWith("大学")) { - if (name.indexOf("校区") == -1 && name.indexOf("幼儿园") == -1 && name.indexOf("小学") == -1 && name.indexOf("中学") == -1 - && !name.endsWith("医院")) { - return false - } - } - if (name.indexOf("学院") != -1 && !name.endsWith("学院")) { - return false - } - if (name.indexOf("分院") != -1 && !name.endsWith("分院")) { - return false - } - if (name.indexOf("学校") != -1 && !name.endsWith("学校")) { - if (name.indexOf("校区") == -1 && name.indexOf("分校") == -1) { - return false - } - } - if (name.indexOf("中学") != -1 && !name.endsWith("中学")) { - if (name.indexOf("校区") == -1) { - return false - } - } - if (name.indexOf("小学") != -1 && !name.endsWith("小学")) { - if (name.indexOf("校区") == -1) { - return false - } - } - } - // 中梁山地铁站1A口 - if (obj.name.indexOf("地铁站") != -1 && obj.name.indexOf("口") != -1) { - def nameTemp = obj.name.substring(0, obj.name.indexOf("地铁站") + 3) - def key = nameTemp - Map child = new HashMap() - if (map.get(key) != null) { - int distance = Double.valueOf(obj.distance) + map.get(key).get("distance") - int count1 = map.get(key).get("count") - child.put("distance", distance) - child.put("count", count1) - map.put(key, child) - } else { - child.put("distance", Double.valueOf(obj.distance)) - child.put("count", 1) - map.put(key, child) - } - } else { - ret.add(obj.name + ":" + obj.distance + "米") } } + + static printUnMatched(def unMatchedMap) { + def ret = [] + ret << ['IDB','城市名称','区域编码','小区名称','楼栋号', '经度', '纬度'] + + for (def r :unMatchedMap.values()) { + ret << r; + } + + com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx", ret, "周边配套"); + } + + static String formatType(String typeStr) { + def types = [] + if (typeStr.indexOf("|") > -1) { + types = typeStr.split("\\|") + } else { + types =[typeStr] + } + for (def type : types) { + if (type.startsWith("09")) { + return "医院"; + } else if (type.startsWith('1505')) { + return "地铁"; + } else if (type.startsWith('1507')) { + return "公交"; + } else if (type.startsWith("16")) { + return "银行"; + } else if (type.startsWith("14")) { + return "学校"; + } else if (type.startsWith("06")) { + return "商场" + } + } + return "未知" + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/CheckLatLngByDistrict.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/CheckLatLngByDistrict.groovy new file mode 100644 index 000000000..a5338340b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/CheckLatLngByDistrict.groovy @@ -0,0 +1,61 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.atomic.AtomicInteger + +/** + * 通过匹配出的小区核对经纬度 + * + * @author shenjian + * @since 2021/5/19 + */ +class CheckLatLngByDistrict { + + static void main(String[] args) { + List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\测试数据汇总_2021.xlsx", 9) + for (List districtT : data) { + final List district = districtT + if (district.size() >24 &&district.get(22) != null && district.get(23) != null && district.get(24) != null) { + DBObject query = new BasicDBObject() + query.put("city", district.get(22)) + query.put("region", district.get(23)) + query.put("cat", "district") + query.put("visible", 0) + query.put("name", district.get(24)) + if (district.get(20) != null && district.get(21) != null) { + def originLng = district.get(20) + def originLat = district.get(21) + DBUtils.seaweed.find(query, new BasicDBObject("lng", 1).append("lat", 1)).limit(1).each { obj -> + if (obj.lng && obj.lat) { + if (originLng != obj.lng || originLat != obj.lat) { + def nowLat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lat / 10000000000d); + def nowLng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lng / 10000000000d); + originLng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, originLng / 10000000000d); + originLat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, originLat / 10000000000d); + + double distance = GaoDeUtils.getDistance(nowLat, nowLng, originLat,originLng) + println(district.get(24) + " 经度纬度不同" + nowLng + "," + nowLat + " 原始" + originLng + "," + originLat + " 距离" + distance + "米" ) + } + } else { + println(district.get(24) + " 不存在经纬度") + } + } + } else { + DBUtils.seaweed.find(query, new BasicDBObject("lng", 1).append("lat", 1)).limit(1).each { obj -> + if (obj.lng && obj.lat) { + + } else { + println(district.get(24) + " 不存在经纬度222222222") + } + } + } + } + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy index 793ff7840..86504ae9a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy @@ -85,8 +85,8 @@ class SeaweedExport { filter.put("lat", 1) DbUtils.seaweed.find(query).limit(1).each { obj -> - ret[0] = obj.developer - ret[1] = obj.estate_company + ret[0] = obj.estate_company + ret[1] = obj.developer ret[2] = SeaweedUtils.formatCubagePercent(obj.cubage_percent) ret[3] = SeaweedUtils.formatVirescencePercent(obj.virescence_percent) ret[4] = SeaweedUtils.formatAlias(obj.alias) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 3e745dfd3..ce6475a95 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -45,7 +45,7 @@ class GaoDeUtils { }}; /** - * + * 查找周边配套 * @param city * @param kw * @param lng @@ -87,10 +87,16 @@ class GaoDeUtils { return result } + /** + * 获取经纬度之间距离 + */ static def getDistance(double Lat, double Lng, double Lat1, double Lng1) { return GeoHelper.GetDistance(Lat, Lng, Lat1, Lng1) } + /** + * 获取地址经纬度 + */ static def parserByGaode(def city, def address) { JSONObject json = GaoDeApiUtils.geo(city, address) if (json && json.status == "1") {