公交去重

This commit is contained in:
沈健
2021-05-19 11:46:43 +08:00
parent e622b12ca7
commit 7726fa48cb
6 changed files with 254 additions and 117 deletions
@@ -1,15 +1,30 @@
package com.fangjia.script.temp.shenjian
import com.fangjia.script.lixue.util.DBUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
/**
* @author shenjian* @since 2021/5/18
*/
class Temp {
static void main(String[] args) {
double val = 1.0
if (val == 1.0) {
println(true)
DBObject query = new BasicDBObject()
query.put("city", "重庆")
query.put("region", "北碚")
query.put("cat", "district")
query.put("visible", 0)
query.put("name", "人民村")
DBUtils.seaweed.find(query).limit(1).each { obj ->
println(obj)
}
}
// 万友七季城C区 经度纬度不同106.569161 29.498502 原始106.56881 29.50058 距离231.7003米
// 马道子 经度纬度不同106.384087 29.66009 原始106.450656 29.550612 距离12306.7728米
// 富洲新城 经度纬度不同106.474475 29.537152 原始106.420021 29.537655 距离1836.9266米
// 滨江国际 经度纬度不同109.466428 31.01454 原始109.46799 31.016907 距离308.3405米
// 澳海澜庭 经度纬度不同109.389067 31.020248 原始109.389069 31.02025 距离0.3029米
// 富洲新城 经度纬度不同106.474475 29.537152 原始106.420021 29.537655 距离1836.9266米
}
@@ -59,11 +59,12 @@ class AroundExportByNewWay {
query.put("cat", "district")
query.put("visible", 0)
query.put("name", district.get(24))
Set<String> set = new HashSet<>()
DBUtils.seaweed.find(query, new BasicDBObject("lng", 1).append("lat", 1)).limit(1).each {obj ->
if (obj.lng && obj.lat) {
def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lng/ 10000000000d)
def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lat / 10000000000d)
calcAroundInfo(lat, lng, result, id)
calcAroundInfo(lat, lng, result, id, set)
}
}
} else {
@@ -106,7 +107,7 @@ class AroundExportByNewWay {
List currRet = resultMap.get(district_code + name)
Collections.sort(currRet)
ret.addAll(currRet)
if (ret.size() > 600000) {
if (ret.size() > 700000) {
map.put(key++, ret)
ret = []
}
@@ -123,7 +124,7 @@ class AroundExportByNewWay {
executorService.shutdown()
}
static calcAroundInfo(def lat, def lng, def result, def id) {
static calcAroundInfo(def lat, def lng, def result, def id, def set) {
// 周围半径为3千米
def mapLatLng = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 3000)
DBObject q = new BasicDBObject()
@@ -136,12 +137,16 @@ class AroundExportByNewWay {
if (distance == 0) {
distance = 50
}
AroundDTO aroundDTO = new AroundDTO()
aroundDTO.id = id
aroundDTO.name = obj.name
aroundDTO.type = formatType(obj.typecode)
aroundDTO.distance = distance.toInteger()
result.add(aroundDTO)
if (!set.contains(obj.name)) {
AroundDTO aroundDTO = new AroundDTO()
aroundDTO.id = id
aroundDTO.name = obj.name
aroundDTO.type = formatType(obj.typecode)
aroundDTO.distance = distance.toInteger()
result.add(aroundDTO)
set.add(obj.name)
}
}
}
}
@@ -1,31 +1,36 @@
package com.fangjia.script.temp.shenjian.cqrcb
import cn.emay.mina.util.ConcurrentHashSet
import com.fangjia.script.coredata.util.CollectionUtils
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.alibaba.excel.EasyExcel
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.district.Distance
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.temp.shenjian.dto.AroundDTO
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.QueryOperators
import org.apache.commons.collections.map.HashedMap
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.ConcurrentLinkedQueue
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
/**
* 小区周围信息格式化输出
* 周边数据一千米配套数量统计
*
* @author shenjian
* @since 2021/5/15
* @since 2021/5/16
*/
class AroundFormatExport {
static DBCollection conn = MongoUtils.getCollectionByName("seeddb_dev", "arround_temp", "192.168.0.235", 27018)
static DBCollection poiConn = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017)
static def resultMap = new ConcurrentHashMap()
static def unMatchedMap = new ConcurrentHashMap()
static ExecutorService executorService = Executors.newFixedThreadPool(30)
static void main(String[] args) {
@@ -38,124 +43,169 @@ class AroundFormatExport {
@Override
void run() {
try {
def id = district.get(0)
def district_code = district.get(2)
def name = district.get(3)
def temp = []
if (district.get(20) != null && district.get(21) != null) {
def matched_city = district.get(22)
def matched_region = district.get(23)
def matched_name = district.get(24)
List result = []
if (district.size() > 21 && district.get(20) != null && district.get(21) != null) {
def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, district.get(20) / 10000000000d)
def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, district.get(21) / 10000000000d)
calcAroundInfo(lat, lng, result, id)
} else if (district.size() >24 &&district.get(22) != null && district.get(23) != null && district.get(24) != null) {
DBObject query = new BasicDBObject()
query.put("matched_city", matched_city)
query.put("matched_region", matched_region)
query.put("matched_name", matched_name)
List ret = new ArrayList()
Map<String, Map> map = new ConcurrentHashMap<>();
conn.find(query, new BasicDBObject("name": 1, "distance": 1, "matched_region": 1)).batchSize(1000).each { obj ->
dealSpecial(obj, map, ret)
query.put("city", district.get(22))
query.put("region", district.get(23))
query.put("cat", "district")
query.put("visible", 0)
query.put("name", district.get(24))
Set<String> set = new HashSet<>()
DBUtils.seaweed.find(query, new BasicDBObject("lng", 1).append("lat", 1)).limit(1).each {obj ->
if (obj.lng && obj.lat) {
def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lng/ 10000000000d)
def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lat / 10000000000d)
calcAroundInfo(lat, lng, result, id, set)
}
}
for (String key : map.keySet()) {
int distance = map.get(key).get("distance")
int count1 = map.get(key).get("count")
ret.add(key + ":" + (distance / count1) + "米")
}
temp[7] = matched_city
temp[8] = matched_region
temp[9] = matched_name
if (ret.size() > 0) {
def str = ret.join(",")
temp[5] = str
println(str)
}
} else {
def ret = []
ret[0] = id
ret[1] = district.get(1)
ret[2] = district_code
ret[3] = name
ret[4] = district.get(4)
unMatchedMap.put(id, ret)
}
resultMap.put(district_code + name, temp)
resultMap.put(district_code + name, result)
println("当前处理" + count.incrementAndGet() )
} catch (Exception ex) {
ex.printStackTrace()
} finally {
latch.countDown()
}
}
})
};
latch.await();
def headArr = ['IDB', '数量']
List<String> head = new ArrayList<>()
for (String name: headArr) {
List<String> temp = new ArrayList<>()
temp.add(name)
head.add(temp)
}
latch.await()
Map<Integer, List> map = new HashedMap()
Integer key = 1;
def ret = []
ret << ['IDB','城市名称','区域编码','小区名称','楼栋号','3km范围内地铁站、公交站、银行、医院、商场、学校等名称及距离信息','1km范围内地铁站、公交站、银行、医院、商场、学校等数量']
AtomicInteger totalCount = new AtomicInteger()
for (List<Object> district : data) {
def id = district.get(0)
def district_code = district.get(2)
def name = district.get(3)
ret << resultMap.get(district_code + name)
if (com.fangjia.script.coredata.util.CollectionUtils.isNotEmpty(resultMap.get(district_code + name))) {
totalCount.incrementAndGet()
List currRet = resultMap.get(district_code + name)
int gj = 0
int dt = 0
int yh = 0
int yy = 0
int sc = 0
int xx = 0
for (AroundDTO dto : currRet) {
if(dto.getType() == '公交') {
gj++
} else if(dto.getType() == '地铁') {
dt++
} else if(dto.getType() == '银行') {
yh++
} else if (dto.getType() == '医院') {
yy++
} else if(dto.getType() == '商场') {
sc++
} else if(dto.getType() == '学校') {
xx++
}
}
ret << [id, "公交:" + gj + ";" + "地铁:" + dt + ";银行:" + yh + ";医院:" + yy + ";商场:" + sc + ";学校:" + xx]
if (ret.size() > 700000) {
map.put(key++, ret)
ret = []
}
} else {
ret << [id, ""]
}
}
com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套.xlsx", ret, "周边配套");
map.put(key++, ret)
//com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套.xlsx", ret, "周边配套");
for (Integer k : map.keySet()) {
//FileUtils.batchCreateFile(map.get(k), "F:\\数据\\重庆农商行\\周边配套"+ k + ".dat", headArr, "utf-8")
EasyExcel.write("F:\\数据\\重庆农商行\\周边配套1千米V1.xlsx").head(head).sheet("周边配套").doWrite(map.get(k));
}
print(totalCount.get() + " " + (totalCount.get() / data.size()))
printUnMatched(unMatchedMap)
executorService.shutdown()
}
static dealSpecial(DBObject obj, def map, def ret) {
def name = obj.name
if (name.indexOf("/") != -1) {
return false
}
if (name == "1号楼") {
return false
}
if (name.indexOf("地铁口") == -1 && name.indexOf("公交站") == -1 && name.indexOf("支行") == -1) {
if (name.indexOf("医院") != -1 && !name.endsWith("医院")) {
if (name.indexOf("分院") == -1) {
return false
static calcAroundInfo(def lat, def lng, def result, def id, def set) {
// 周围半径为3千米
def mapLatLng = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 1000)
DBObject q = new BasicDBObject()
q.put('status', 0)
q.put('lat', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLat).append(QueryOperators.GTE, mapLatLng.minLat));
q.put('lng', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLng).append(QueryOperators.GTE, mapLatLng.minLng));
poiConn.find(q, new BasicDBObject("name": 1, "distance": 1, "type": 1, "typecode": 1, "lat": 1, "lng": 1)).batchSize(500).each {obj ->
double distance = Distance.GetDistance(lng, lat, obj.lng, obj.lat)
if (distance <= 1000) {
if (distance == 0) {
distance = 50
}
if (!set.contains(obj.name)) {
AroundDTO aroundDTO = new AroundDTO()
aroundDTO.id = id
aroundDTO.name = obj.name
aroundDTO.type = formatType(obj.typecode)
aroundDTO.distance = distance.toInteger()
result.add(aroundDTO)
set.add(obj.name)
}
}
if (name.indexOf("卫生中心") != -1 && !name.endsWith("卫生中心")) {
return false
}
if (name.indexOf("大学") != -1 && !name.endsWith("大学")) {
if (name.indexOf("校区") == -1 && name.indexOf("幼儿园") == -1 && name.indexOf("小学") == -1 && name.indexOf("中学") == -1
&& !name.endsWith("医院")) {
return false
}
}
if (name.indexOf("学院") != -1 && !name.endsWith("学院")) {
return false
}
if (name.indexOf("分院") != -1 && !name.endsWith("分院")) {
return false
}
if (name.indexOf("学校") != -1 && !name.endsWith("学校")) {
if (name.indexOf("校区") == -1 && name.indexOf("分校") == -1) {
return false
}
}
if (name.indexOf("中学") != -1 && !name.endsWith("中学")) {
if (name.indexOf("校区") == -1) {
return false
}
}
if (name.indexOf("小学") != -1 && !name.endsWith("小学")) {
if (name.indexOf("校区") == -1) {
return false
}
}
}
// 中梁山地铁站1A口
if (obj.name.indexOf("地铁站") != -1 && obj.name.indexOf("口") != -1) {
def nameTemp = obj.name.substring(0, obj.name.indexOf("地铁站") + 3)
def key = nameTemp
Map<String, Integer> child = new HashMap()
if (map.get(key) != null) {
int distance = Double.valueOf(obj.distance) + map.get(key).get("distance")
int count1 = map.get(key).get("count")
child.put("distance", distance)
child.put("count", count1)
map.put(key, child)
} else {
child.put("distance", Double.valueOf(obj.distance))
child.put("count", 1)
map.put(key, child)
}
} else {
ret.add(obj.name + ":" + obj.distance + "米")
}
}
static printUnMatched(def unMatchedMap) {
def ret = []
ret << ['IDB','城市名称','区域编码','小区名称','楼栋号', '经度', '纬度']
for (def r :unMatchedMap.values()) {
ret << r;
}
com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx", ret, "周边配套");
}
static String formatType(String typeStr) {
def types = []
if (typeStr.indexOf("|") > -1) {
types = typeStr.split("\\|")
} else {
types =[typeStr]
}
for (def type : types) {
if (type.startsWith("09")) {
return "医院";
} else if (type.startsWith('1505')) {
return "地铁";
} else if (type.startsWith('1507')) {
return "公交";
} else if (type.startsWith("16")) {
return "银行";
} else if (type.startsWith("14")) {
return "学校";
} else if (type.startsWith("06")) {
return "商场"
}
}
return "未知"
}
}
@@ -0,0 +1,61 @@
package com.fangjia.script.temp.shenjian.cqrcb
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import java.util.concurrent.CountDownLatch
import java.util.concurrent.atomic.AtomicInteger
/**
* 通过匹配出的小区核对经纬度
*
* @author shenjian
* @since 2021/5/19
*/
class CheckLatLngByDistrict {
static void main(String[] args) {
List<List<Object>> data = ExcelUtils.read("F:\\数据\\重庆农商行\\测试数据汇总_2021.xlsx", 9)
for (List<Object> districtT : data) {
final List<Object> district = districtT
if (district.size() >24 &&district.get(22) != null && district.get(23) != null && district.get(24) != null) {
DBObject query = new BasicDBObject()
query.put("city", district.get(22))
query.put("region", district.get(23))
query.put("cat", "district")
query.put("visible", 0)
query.put("name", district.get(24))
if (district.get(20) != null && district.get(21) != null) {
def originLng = district.get(20)
def originLat = district.get(21)
DBUtils.seaweed.find(query, new BasicDBObject("lng", 1).append("lat", 1)).limit(1).each { obj ->
if (obj.lng && obj.lat) {
if (originLng != obj.lng || originLat != obj.lat) {
def nowLat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lat / 10000000000d);
def nowLng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lng / 10000000000d);
originLng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, originLng / 10000000000d);
originLat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, originLat / 10000000000d);
double distance = GaoDeUtils.getDistance(nowLat, nowLng, originLat,originLng)
println(district.get(24) + " 经度纬度不同" + nowLng + "," + nowLat + " 原始" + originLng + "," + originLat + " 距离" + distance + "米" )
}
} else {
println(district.get(24) + " 不存在经纬度")
}
}
} else {
DBUtils.seaweed.find(query, new BasicDBObject("lng", 1).append("lat", 1)).limit(1).each { obj ->
if (obj.lng && obj.lat) {
} else {
println(district.get(24) + " 不存在经纬度222222222")
}
}
}
}
}
}
}
@@ -85,8 +85,8 @@ class SeaweedExport {
filter.put("lat", 1)
DbUtils.seaweed.find(query).limit(1).each { obj ->
ret[0] = obj.developer
ret[1] = obj.estate_company
ret[0] = obj.estate_company
ret[1] = obj.developer
ret[2] = SeaweedUtils.formatCubagePercent(obj.cubage_percent)
ret[3] = SeaweedUtils.formatVirescencePercent(obj.virescence_percent)
ret[4] = SeaweedUtils.formatAlias(obj.alias)
@@ -45,7 +45,7 @@ class GaoDeUtils {
}};
/**
*
* 查找周边配套
* @param city
* @param kw
* @param lng
@@ -87,10 +87,16 @@ class GaoDeUtils {
return result
}
/**
* 获取经纬度之间距离
*/
static def getDistance(double Lat, double Lng, double Lat1, double Lng1) {
return GeoHelper.GetDistance(Lat, Lng, Lat1, Lng1)
}
/**
* 获取地址经纬度
*/
static def parserByGaode(def city, def address) {
JSONObject json = GaoDeApiUtils.geo(city, address)
if (json && json.status == "1") {