公交去重

This commit is contained in:
沈健
2021-05-19 11:46:43 +08:00
parent e622b12ca7
commit 7726fa48cb
6 changed files with 254 additions and 117 deletions
@@ -1,15 +1,30 @@
package com.fangjia.script.temp.shenjian package com.fangjia.script.temp.shenjian
import com.fangjia.script.lixue.util.DBUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
/** /**
* @author shenjian* @since 2021/5/18 * @author shenjian* @since 2021/5/18
*/ */
class Temp { class Temp {
static void main(String[] args) { static void main(String[] args) {
double val = 1.0 DBObject query = new BasicDBObject()
if (val == 1.0) { query.put("city", "重庆")
println(true) query.put("region", "北碚")
query.put("cat", "district")
query.put("visible", 0)
query.put("name", "人民村")
DBUtils.seaweed.find(query).limit(1).each { obj ->
println(obj)
} }
} }
// 万友七季城C区 经度纬度不同106.569161 29.498502 原始106.56881 29.50058 距离231.7003米
// 马道子 经度纬度不同106.384087 29.66009 原始106.450656 29.550612 距离12306.7728米
// 富洲新城 经度纬度不同106.474475 29.537152 原始106.420021 29.537655 距离1836.9266米
// 滨江国际 经度纬度不同109.466428 31.01454 原始109.46799 31.016907 距离308.3405米
// 澳海澜庭 经度纬度不同109.389067 31.020248 原始109.389069 31.02025 距离0.3029米
// 富洲新城 经度纬度不同106.474475 29.537152 原始106.420021 29.537655 距离1836.9266米
} }
@@ -59,11 +59,12 @@ class AroundExportByNewWay {
query.put("cat", "district") query.put("cat", "district")
query.put("visible", 0) query.put("visible", 0)
query.put("name", district.get(24)) query.put("name", district.get(24))
Set<String> set = new HashSet<>()
DBUtils.seaweed.find(query, new BasicDBObject("lng", 1).append("lat", 1)).limit(1).each {obj -> DBUtils.seaweed.find(query, new BasicDBObject("lng", 1).append("lat", 1)).limit(1).each {obj ->
if (obj.lng && obj.lat) { if (obj.lng && obj.lat) {
def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lng/ 10000000000d) def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lng/ 10000000000d)
def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lat / 10000000000d) def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lat / 10000000000d)
calcAroundInfo(lat, lng, result, id) calcAroundInfo(lat, lng, result, id, set)
} }
} }
} else { } else {
@@ -106,7 +107,7 @@ class AroundExportByNewWay {
List currRet = resultMap.get(district_code + name) List currRet = resultMap.get(district_code + name)
Collections.sort(currRet) Collections.sort(currRet)
ret.addAll(currRet) ret.addAll(currRet)
if (ret.size() > 600000) { if (ret.size() > 700000) {
map.put(key++, ret) map.put(key++, ret)
ret = [] ret = []
} }
@@ -123,7 +124,7 @@ class AroundExportByNewWay {
executorService.shutdown() executorService.shutdown()
} }
static calcAroundInfo(def lat, def lng, def result, def id) { static calcAroundInfo(def lat, def lng, def result, def id, def set) {
// 周围半径为3千米 // 周围半径为3千米
def mapLatLng = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 3000) def mapLatLng = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 3000)
DBObject q = new BasicDBObject() DBObject q = new BasicDBObject()
@@ -136,12 +137,16 @@ class AroundExportByNewWay {
if (distance == 0) { if (distance == 0) {
distance = 50 distance = 50
} }
AroundDTO aroundDTO = new AroundDTO() if (!set.contains(obj.name)) {
aroundDTO.id = id AroundDTO aroundDTO = new AroundDTO()
aroundDTO.name = obj.name aroundDTO.id = id
aroundDTO.type = formatType(obj.typecode) aroundDTO.name = obj.name
aroundDTO.distance = distance.toInteger() aroundDTO.type = formatType(obj.typecode)
result.add(aroundDTO) aroundDTO.distance = distance.toInteger()
result.add(aroundDTO)
set.add(obj.name)
}
} }
} }
} }
@@ -1,31 +1,36 @@
package com.fangjia.script.temp.shenjian.cqrcb package com.fangjia.script.temp.shenjian.cqrcb
import cn.emay.mina.util.ConcurrentHashSet import com.alibaba.excel.EasyExcel
import com.fangjia.script.coredata.util.CollectionUtils import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.temp.shenjian.util.GaoDeUtils import com.fangjia.script.district.Distance
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.temp.shenjian.dto.AroundDTO
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.DBCollection import com.mongodb.DBCollection
import com.mongodb.DBObject import com.mongodb.DBObject
import com.mongodb.QueryOperators
import org.apache.commons.collections.map.HashedMap
import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.ConcurrentLinkedQueue
import java.util.concurrent.CountDownLatch import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger import java.util.concurrent.atomic.AtomicInteger
/** /**
* 小区周围信息格式化输出 * 周边数据一千米配套数量统计
* *
* @author shenjian * @author shenjian
* @since 2021/5/15 * @since 2021/5/16
*/ */
class AroundFormatExport { class AroundFormatExport {
static DBCollection conn = MongoUtils.getCollectionByName("seeddb_dev", "arround_temp", "192.168.0.235", 27018) static DBCollection poiConn = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017)
static def resultMap = new ConcurrentHashMap() static def resultMap = new ConcurrentHashMap()
static def unMatchedMap = new ConcurrentHashMap()
static ExecutorService executorService = Executors.newFixedThreadPool(30) static ExecutorService executorService = Executors.newFixedThreadPool(30)
static void main(String[] args) { static void main(String[] args) {
@@ -38,124 +43,169 @@ class AroundFormatExport {
@Override @Override
void run() { void run() {
try { try {
def id = district.get(0)
def district_code = district.get(2) def district_code = district.get(2)
def name = district.get(3) def name = district.get(3)
def temp = [] List result = []
if (district.get(20) != null && district.get(21) != null) { if (district.size() > 21 && district.get(20) != null && district.get(21) != null) {
def matched_city = district.get(22) def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, district.get(20) / 10000000000d)
def matched_region = district.get(23) def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, district.get(21) / 10000000000d)
def matched_name = district.get(24) calcAroundInfo(lat, lng, result, id)
} else if (district.size() >24 &&district.get(22) != null && district.get(23) != null && district.get(24) != null) {
DBObject query = new BasicDBObject() DBObject query = new BasicDBObject()
query.put("matched_city", matched_city) query.put("city", district.get(22))
query.put("matched_region", matched_region) query.put("region", district.get(23))
query.put("matched_name", matched_name) query.put("cat", "district")
List ret = new ArrayList() query.put("visible", 0)
Map<String, Map> map = new ConcurrentHashMap<>(); query.put("name", district.get(24))
conn.find(query, new BasicDBObject("name": 1, "distance": 1, "matched_region": 1)).batchSize(1000).each { obj -> Set<String> set = new HashSet<>()
dealSpecial(obj, map, ret) DBUtils.seaweed.find(query, new BasicDBObject("lng", 1).append("lat", 1)).limit(1).each {obj ->
if (obj.lng && obj.lat) {
def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lng/ 10000000000d)
def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lat / 10000000000d)
calcAroundInfo(lat, lng, result, id, set)
}
} }
for (String key : map.keySet()) { } else {
int distance = map.get(key).get("distance") def ret = []
int count1 = map.get(key).get("count") ret[0] = id
ret.add(key + ":" + (distance / count1) + "米") ret[1] = district.get(1)
} ret[2] = district_code
ret[3] = name
temp[7] = matched_city ret[4] = district.get(4)
temp[8] = matched_region unMatchedMap.put(id, ret)
temp[9] = matched_name
if (ret.size() > 0) {
def str = ret.join(",")
temp[5] = str
println(str)
}
} }
resultMap.put(district_code + name, temp) resultMap.put(district_code + name, result)
println("当前处理" + count.incrementAndGet() ) println("当前处理" + count.incrementAndGet() )
} catch (Exception ex) { } catch (Exception ex) {
ex.printStackTrace()
} finally { } finally {
latch.countDown() latch.countDown()
} }
} }
}) })
};
latch.await();
def headArr = ['IDB', '数量']
List<String> head = new ArrayList<>()
for (String name: headArr) {
List<String> temp = new ArrayList<>()
temp.add(name)
head.add(temp)
} }
latch.await() Map<Integer, List> map = new HashedMap()
Integer key = 1;
def ret = [] def ret = []
ret << ['IDB','城市名称','区域编码','小区名称','楼栋号','3km范围内地铁站、公交站、银行、医院、商场、学校等名称及距离信息','1km范围内地铁站、公交站、银行、医院、商场、学校等数量'] AtomicInteger totalCount = new AtomicInteger()
for (List<Object> district : data) { for (List<Object> district : data) {
def id = district.get(0)
def district_code = district.get(2) def district_code = district.get(2)
def name = district.get(3) def name = district.get(3)
ret << resultMap.get(district_code + name) if (com.fangjia.script.coredata.util.CollectionUtils.isNotEmpty(resultMap.get(district_code + name))) {
totalCount.incrementAndGet()
List currRet = resultMap.get(district_code + name)
int gj = 0
int dt = 0
int yh = 0
int yy = 0
int sc = 0
int xx = 0
for (AroundDTO dto : currRet) {
if(dto.getType() == '公交') {
gj++
} else if(dto.getType() == '地铁') {
dt++
} else if(dto.getType() == '银行') {
yh++
} else if (dto.getType() == '医院') {
yy++
} else if(dto.getType() == '商场') {
sc++
} else if(dto.getType() == '学校') {
xx++
}
}
ret << [id, "公交:" + gj + ";" + "地铁:" + dt + ";银行:" + yh + ";医院:" + yy + ";商场:" + sc + ";学校:" + xx]
if (ret.size() > 700000) {
map.put(key++, ret)
ret = []
}
} else {
ret << [id, ""]
}
} }
com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套.xlsx", ret, "周边配套"); map.put(key++, ret)
//com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套.xlsx", ret, "周边配套");
for (Integer k : map.keySet()) {
//FileUtils.batchCreateFile(map.get(k), "F:\\数据\\重庆农商行\\周边配套"+ k + ".dat", headArr, "utf-8")
EasyExcel.write("F:\\数据\\重庆农商行\\周边配套1千米V1.xlsx").head(head).sheet("周边配套").doWrite(map.get(k));
}
print(totalCount.get() + " " + (totalCount.get() / data.size()))
printUnMatched(unMatchedMap)
executorService.shutdown() executorService.shutdown()
} }
static dealSpecial(DBObject obj, def map, def ret) { static calcAroundInfo(def lat, def lng, def result, def id, def set) {
def name = obj.name // 周围半径为3千米
if (name.indexOf("/") != -1) { def mapLatLng = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 1000)
return false DBObject q = new BasicDBObject()
} q.put('status', 0)
if (name == "1号楼") { q.put('lat', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLat).append(QueryOperators.GTE, mapLatLng.minLat));
return false q.put('lng', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLng).append(QueryOperators.GTE, mapLatLng.minLng));
} poiConn.find(q, new BasicDBObject("name": 1, "distance": 1, "type": 1, "typecode": 1, "lat": 1, "lng": 1)).batchSize(500).each {obj ->
if (name.indexOf("地铁口") == -1 && name.indexOf("公交站") == -1 && name.indexOf("支行") == -1) { double distance = Distance.GetDistance(lng, lat, obj.lng, obj.lat)
if (name.indexOf("医院") != -1 && !name.endsWith("医院")) { if (distance <= 1000) {
if (name.indexOf("分院") == -1) { if (distance == 0) {
return false distance = 50
}
if (!set.contains(obj.name)) {
AroundDTO aroundDTO = new AroundDTO()
aroundDTO.id = id
aroundDTO.name = obj.name
aroundDTO.type = formatType(obj.typecode)
aroundDTO.distance = distance.toInteger()
result.add(aroundDTO)
set.add(obj.name)
} }
} }
if (name.indexOf("卫生中心") != -1 && !name.endsWith("卫生中心")) {
return false
}
if (name.indexOf("大学") != -1 && !name.endsWith("大学")) {
if (name.indexOf("校区") == -1 && name.indexOf("幼儿园") == -1 && name.indexOf("小学") == -1 && name.indexOf("中学") == -1
&& !name.endsWith("医院")) {
return false
}
}
if (name.indexOf("学院") != -1 && !name.endsWith("学院")) {
return false
}
if (name.indexOf("分院") != -1 && !name.endsWith("分院")) {
return false
}
if (name.indexOf("学校") != -1 && !name.endsWith("学校")) {
if (name.indexOf("校区") == -1 && name.indexOf("分校") == -1) {
return false
}
}
if (name.indexOf("中学") != -1 && !name.endsWith("中学")) {
if (name.indexOf("校区") == -1) {
return false
}
}
if (name.indexOf("小学") != -1 && !name.endsWith("小学")) {
if (name.indexOf("校区") == -1) {
return false
}
}
}
// 中梁山地铁站1A口
if (obj.name.indexOf("地铁站") != -1 && obj.name.indexOf("口") != -1) {
def nameTemp = obj.name.substring(0, obj.name.indexOf("地铁站") + 3)
def key = nameTemp
Map<String, Integer> child = new HashMap()
if (map.get(key) != null) {
int distance = Double.valueOf(obj.distance) + map.get(key).get("distance")
int count1 = map.get(key).get("count")
child.put("distance", distance)
child.put("count", count1)
map.put(key, child)
} else {
child.put("distance", Double.valueOf(obj.distance))
child.put("count", 1)
map.put(key, child)
}
} else {
ret.add(obj.name + ":" + obj.distance + "米")
} }
} }
static printUnMatched(def unMatchedMap) {
def ret = []
ret << ['IDB','城市名称','区域编码','小区名称','楼栋号', '经度', '纬度']
for (def r :unMatchedMap.values()) {
ret << r;
}
com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx", ret, "周边配套");
}
static String formatType(String typeStr) {
def types = []
if (typeStr.indexOf("|") > -1) {
types = typeStr.split("\\|")
} else {
types =[typeStr]
}
for (def type : types) {
if (type.startsWith("09")) {
return "医院";
} else if (type.startsWith('1505')) {
return "地铁";
} else if (type.startsWith('1507')) {
return "公交";
} else if (type.startsWith("16")) {
return "银行";
} else if (type.startsWith("14")) {
return "学校";
} else if (type.startsWith("06")) {
return "商场"
}
}
return "未知"
}
} }
@@ -0,0 +1,61 @@
package com.fangjia.script.temp.shenjian.cqrcb
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import java.util.concurrent.CountDownLatch
import java.util.concurrent.atomic.AtomicInteger
/**
* 通过匹配出的小区核对经纬度
*
* @author shenjian
* @since 2021/5/19
*/
class CheckLatLngByDistrict {
static void main(String[] args) {
List<List<Object>> data = ExcelUtils.read("F:\\数据\\重庆农商行\\测试数据汇总_2021.xlsx", 9)
for (List<Object> districtT : data) {
final List<Object> district = districtT
if (district.size() >24 &&district.get(22) != null && district.get(23) != null && district.get(24) != null) {
DBObject query = new BasicDBObject()
query.put("city", district.get(22))
query.put("region", district.get(23))
query.put("cat", "district")
query.put("visible", 0)
query.put("name", district.get(24))
if (district.get(20) != null && district.get(21) != null) {
def originLng = district.get(20)
def originLat = district.get(21)
DBUtils.seaweed.find(query, new BasicDBObject("lng", 1).append("lat", 1)).limit(1).each { obj ->
if (obj.lng && obj.lat) {
if (originLng != obj.lng || originLat != obj.lat) {
def nowLat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lat / 10000000000d);
def nowLng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lng / 10000000000d);
originLng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, originLng / 10000000000d);
originLat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, originLat / 10000000000d);
double distance = GaoDeUtils.getDistance(nowLat, nowLng, originLat,originLng)
println(district.get(24) + " 经度纬度不同" + nowLng + "," + nowLat + " 原始" + originLng + "," + originLat + " 距离" + distance + "米" )
}
} else {
println(district.get(24) + " 不存在经纬度")
}
}
} else {
DBUtils.seaweed.find(query, new BasicDBObject("lng", 1).append("lat", 1)).limit(1).each { obj ->
if (obj.lng && obj.lat) {
} else {
println(district.get(24) + " 不存在经纬度222222222")
}
}
}
}
}
}
}
@@ -85,8 +85,8 @@ class SeaweedExport {
filter.put("lat", 1) filter.put("lat", 1)
DbUtils.seaweed.find(query).limit(1).each { obj -> DbUtils.seaweed.find(query).limit(1).each { obj ->
ret[0] = obj.developer ret[0] = obj.estate_company
ret[1] = obj.estate_company ret[1] = obj.developer
ret[2] = SeaweedUtils.formatCubagePercent(obj.cubage_percent) ret[2] = SeaweedUtils.formatCubagePercent(obj.cubage_percent)
ret[3] = SeaweedUtils.formatVirescencePercent(obj.virescence_percent) ret[3] = SeaweedUtils.formatVirescencePercent(obj.virescence_percent)
ret[4] = SeaweedUtils.formatAlias(obj.alias) ret[4] = SeaweedUtils.formatAlias(obj.alias)
@@ -45,7 +45,7 @@ class GaoDeUtils {
}}; }};
/** /**
* * 查找周边配套
* @param city * @param city
* @param kw * @param kw
* @param lng * @param lng
@@ -87,10 +87,16 @@ class GaoDeUtils {
return result return result
} }
/**
* 获取经纬度之间距离
*/
static def getDistance(double Lat, double Lng, double Lat1, double Lng1) { static def getDistance(double Lat, double Lng, double Lat1, double Lng1) {
return GeoHelper.GetDistance(Lat, Lng, Lat1, Lng1) return GeoHelper.GetDistance(Lat, Lng, Lat1, Lng1)
} }
/**
* 获取地址经纬度
*/
static def parserByGaode(def city, def address) { static def parserByGaode(def city, def address) {
JSONObject json = GaoDeApiUtils.geo(city, address) JSONObject json = GaoDeApiUtils.geo(city, address)
if (json && json.status == "1") { if (json && json.status == "1") {