公交去重
This commit is contained in:
+19
-4
@@ -1,15 +1,30 @@
|
||||
package com.fangjia.script.temp.shenjian
|
||||
|
||||
import com.fangjia.script.lixue.util.DBUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBObject
|
||||
|
||||
/**
|
||||
* @author shenjian* @since 2021/5/18
|
||||
*/
|
||||
class Temp {
|
||||
|
||||
static void main(String[] args) {
|
||||
double val = 1.0
|
||||
if (val == 1.0) {
|
||||
println(true)
|
||||
DBObject query = new BasicDBObject()
|
||||
query.put("city", "重庆")
|
||||
query.put("region", "北碚")
|
||||
query.put("cat", "district")
|
||||
query.put("visible", 0)
|
||||
query.put("name", "人民村")
|
||||
DBUtils.seaweed.find(query).limit(1).each { obj ->
|
||||
println(obj)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// 万友七季城C区 经度纬度不同106.569161 29.498502 原始106.56881 29.50058 距离231.7003米
|
||||
// 马道子 经度纬度不同106.384087 29.66009 原始106.450656 29.550612 距离12306.7728米
|
||||
// 富洲新城 经度纬度不同106.474475 29.537152 原始106.420021 29.537655 距离1836.9266米
|
||||
// 滨江国际 经度纬度不同109.466428 31.01454 原始109.46799 31.016907 距离308.3405米
|
||||
// 澳海澜庭 经度纬度不同109.389067 31.020248 原始109.389069 31.02025 距离0.3029米
|
||||
// 富洲新城 经度纬度不同106.474475 29.537152 原始106.420021 29.537655 距离1836.9266米
|
||||
}
|
||||
|
||||
+14
-9
@@ -59,11 +59,12 @@ class AroundExportByNewWay {
|
||||
query.put("cat", "district")
|
||||
query.put("visible", 0)
|
||||
query.put("name", district.get(24))
|
||||
Set<String> set = new HashSet<>()
|
||||
DBUtils.seaweed.find(query, new BasicDBObject("lng", 1).append("lat", 1)).limit(1).each {obj ->
|
||||
if (obj.lng && obj.lat) {
|
||||
def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lng/ 10000000000d)
|
||||
def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lat / 10000000000d)
|
||||
calcAroundInfo(lat, lng, result, id)
|
||||
calcAroundInfo(lat, lng, result, id, set)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
@@ -106,7 +107,7 @@ class AroundExportByNewWay {
|
||||
List currRet = resultMap.get(district_code + name)
|
||||
Collections.sort(currRet)
|
||||
ret.addAll(currRet)
|
||||
if (ret.size() > 600000) {
|
||||
if (ret.size() > 700000) {
|
||||
map.put(key++, ret)
|
||||
ret = []
|
||||
}
|
||||
@@ -123,7 +124,7 @@ class AroundExportByNewWay {
|
||||
executorService.shutdown()
|
||||
}
|
||||
|
||||
static calcAroundInfo(def lat, def lng, def result, def id) {
|
||||
static calcAroundInfo(def lat, def lng, def result, def id, def set) {
|
||||
// 周围半径为3千米
|
||||
def mapLatLng = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 3000)
|
||||
DBObject q = new BasicDBObject()
|
||||
@@ -136,12 +137,16 @@ class AroundExportByNewWay {
|
||||
if (distance == 0) {
|
||||
distance = 50
|
||||
}
|
||||
AroundDTO aroundDTO = new AroundDTO()
|
||||
aroundDTO.id = id
|
||||
aroundDTO.name = obj.name
|
||||
aroundDTO.type = formatType(obj.typecode)
|
||||
aroundDTO.distance = distance.toInteger()
|
||||
result.add(aroundDTO)
|
||||
if (!set.contains(obj.name)) {
|
||||
AroundDTO aroundDTO = new AroundDTO()
|
||||
aroundDTO.id = id
|
||||
aroundDTO.name = obj.name
|
||||
aroundDTO.type = formatType(obj.typecode)
|
||||
aroundDTO.distance = distance.toInteger()
|
||||
result.add(aroundDTO)
|
||||
|
||||
set.add(obj.name)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+130
-101
@@ -1,31 +1,36 @@
|
||||
package com.fangjia.script.temp.shenjian.cqrcb
|
||||
|
||||
import cn.emay.mina.util.ConcurrentHashSet
|
||||
import com.fangjia.script.coredata.util.CollectionUtils
|
||||
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
|
||||
import com.alibaba.excel.EasyExcel
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.district.Distance
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.lixue.util.DBUtils
|
||||
import com.fangjia.script.temp.shenjian.dto.AroundDTO
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.QueryOperators
|
||||
import org.apache.commons.collections.map.HashedMap
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.ConcurrentLinkedQueue
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
|
||||
/**
|
||||
* 小区周围信息格式化输出
|
||||
* 周边数据一千米配套数量统计
|
||||
*
|
||||
* @author shenjian
|
||||
* @since 2021/5/15
|
||||
* @since 2021/5/16
|
||||
*/
|
||||
class AroundFormatExport {
|
||||
|
||||
static DBCollection conn = MongoUtils.getCollectionByName("seeddb_dev", "arround_temp", "192.168.0.235", 27018)
|
||||
static DBCollection poiConn = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017)
|
||||
static def resultMap = new ConcurrentHashMap()
|
||||
static def unMatchedMap = new ConcurrentHashMap()
|
||||
static ExecutorService executorService = Executors.newFixedThreadPool(30)
|
||||
|
||||
static void main(String[] args) {
|
||||
@@ -38,124 +43,148 @@ class AroundFormatExport {
|
||||
@Override
|
||||
void run() {
|
||||
try {
|
||||
def id = district.get(0)
|
||||
def district_code = district.get(2)
|
||||
def name = district.get(3)
|
||||
def temp = []
|
||||
if (district.get(20) != null && district.get(21) != null) {
|
||||
def matched_city = district.get(22)
|
||||
def matched_region = district.get(23)
|
||||
def matched_name = district.get(24)
|
||||
List result = []
|
||||
if (district.size() > 21 && district.get(20) != null && district.get(21) != null) {
|
||||
def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, district.get(20) / 10000000000d)
|
||||
def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, district.get(21) / 10000000000d)
|
||||
calcAroundInfo(lat, lng, result, id)
|
||||
|
||||
} else if (district.size() >24 &&district.get(22) != null && district.get(23) != null && district.get(24) != null) {
|
||||
DBObject query = new BasicDBObject()
|
||||
query.put("matched_city", matched_city)
|
||||
query.put("matched_region", matched_region)
|
||||
query.put("matched_name", matched_name)
|
||||
List ret = new ArrayList()
|
||||
Map<String, Map> map = new ConcurrentHashMap<>();
|
||||
conn.find(query, new BasicDBObject("name": 1, "distance": 1, "matched_region": 1)).batchSize(1000).each { obj ->
|
||||
dealSpecial(obj, map, ret)
|
||||
query.put("city", district.get(22))
|
||||
query.put("region", district.get(23))
|
||||
query.put("cat", "district")
|
||||
query.put("visible", 0)
|
||||
query.put("name", district.get(24))
|
||||
Set<String> set = new HashSet<>()
|
||||
DBUtils.seaweed.find(query, new BasicDBObject("lng", 1).append("lat", 1)).limit(1).each {obj ->
|
||||
if (obj.lng && obj.lat) {
|
||||
def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lng/ 10000000000d)
|
||||
def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lat / 10000000000d)
|
||||
calcAroundInfo(lat, lng, result, id, set)
|
||||
}
|
||||
}
|
||||
for (String key : map.keySet()) {
|
||||
int distance = map.get(key).get("distance")
|
||||
int count1 = map.get(key).get("count")
|
||||
ret.add(key + ":" + (distance / count1) + "米")
|
||||
}
|
||||
|
||||
temp[7] = matched_city
|
||||
temp[8] = matched_region
|
||||
temp[9] = matched_name
|
||||
if (ret.size() > 0) {
|
||||
def str = ret.join(",")
|
||||
temp[5] = str
|
||||
println(str)
|
||||
}
|
||||
|
||||
} else {
|
||||
def ret = []
|
||||
ret[0] = id
|
||||
ret[1] = district.get(1)
|
||||
ret[2] = district_code
|
||||
ret[3] = name
|
||||
ret[4] = district.get(4)
|
||||
unMatchedMap.put(id, ret)
|
||||
}
|
||||
resultMap.put(district_code + name, temp)
|
||||
resultMap.put(district_code + name, result)
|
||||
println("当前处理" + count.incrementAndGet() )
|
||||
} catch (Exception ex) {
|
||||
|
||||
ex.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
})
|
||||
};
|
||||
latch.await();
|
||||
def headArr = ['IDB', '数量']
|
||||
List<String> head = new ArrayList<>()
|
||||
for (String name: headArr) {
|
||||
List<String> temp = new ArrayList<>()
|
||||
temp.add(name)
|
||||
head.add(temp)
|
||||
}
|
||||
latch.await()
|
||||
Map<Integer, List> map = new HashedMap()
|
||||
Integer key = 1;
|
||||
def ret = []
|
||||
ret << ['IDB','城市名称','区域编码','小区名称','楼栋号','3km范围内地铁站、公交站、银行、医院、商场、学校等名称及距离信息','1km范围内地铁站、公交站、银行、医院、商场、学校等数量']
|
||||
AtomicInteger totalCount = new AtomicInteger()
|
||||
for (List<Object> district : data) {
|
||||
def id = district.get(0)
|
||||
def district_code = district.get(2)
|
||||
def name = district.get(3)
|
||||
ret << resultMap.get(district_code + name)
|
||||
if (com.fangjia.script.coredata.util.CollectionUtils.isNotEmpty(resultMap.get(district_code + name))) {
|
||||
totalCount.incrementAndGet()
|
||||
List currRet = resultMap.get(district_code + name)
|
||||
ret << [id, currRet.size()]
|
||||
if (ret.size() > 700000) {
|
||||
map.put(key++, ret)
|
||||
ret = []
|
||||
}
|
||||
} else {
|
||||
ret << [id, 0]
|
||||
}
|
||||
}
|
||||
com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套.xlsx", ret, "周边配套");
|
||||
map.put(key++, ret)
|
||||
//com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套.xlsx", ret, "周边配套");
|
||||
for (Integer k : map.keySet()) {
|
||||
//FileUtils.batchCreateFile(map.get(k), "F:\\数据\\重庆农商行\\周边配套"+ k + ".dat", headArr, "utf-8")
|
||||
EasyExcel.write("F:\\数据\\重庆农商行\\周边配套1千米V1.xlsx").head(head).sheet("周边配套").doWrite(map.get(k));
|
||||
}
|
||||
print(totalCount.get() + " " + (totalCount.get() / data.size()))
|
||||
printUnMatched(unMatchedMap)
|
||||
executorService.shutdown()
|
||||
}
|
||||
|
||||
static dealSpecial(DBObject obj, def map, def ret) {
|
||||
def name = obj.name
|
||||
if (name.indexOf("/") != -1) {
|
||||
return false
|
||||
}
|
||||
if (name == "1号楼") {
|
||||
return false
|
||||
}
|
||||
if (name.indexOf("地铁口") == -1 && name.indexOf("公交站") == -1 && name.indexOf("支行") == -1) {
|
||||
if (name.indexOf("医院") != -1 && !name.endsWith("医院")) {
|
||||
if (name.indexOf("分院") == -1) {
|
||||
return false
|
||||
static calcAroundInfo(def lat, def lng, def result, def id, def set) {
|
||||
// 周围半径为3千米
|
||||
def mapLatLng = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 1000)
|
||||
DBObject q = new BasicDBObject()
|
||||
q.put('status', 0)
|
||||
q.put('lat', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLat).append(QueryOperators.GTE, mapLatLng.minLat));
|
||||
q.put('lng', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLng).append(QueryOperators.GTE, mapLatLng.minLng));
|
||||
poiConn.find(q, new BasicDBObject("name": 1, "distance": 1, "type": 1, "typecode": 1, "lat": 1, "lng": 1)).batchSize(500).each {obj ->
|
||||
double distance = Distance.GetDistance(lng, lat, obj.lng, obj.lat)
|
||||
if (distance <= 1000) {
|
||||
if (distance == 0) {
|
||||
distance = 50
|
||||
}
|
||||
if (!set.contains(obj.name)) {
|
||||
AroundDTO aroundDTO = new AroundDTO()
|
||||
aroundDTO.id = id
|
||||
aroundDTO.name = obj.name
|
||||
aroundDTO.type = formatType(obj.typecode)
|
||||
aroundDTO.distance = distance.toInteger()
|
||||
result.add(aroundDTO)
|
||||
|
||||
set.add(obj.name)
|
||||
}
|
||||
}
|
||||
if (name.indexOf("卫生中心") != -1 && !name.endsWith("卫生中心")) {
|
||||
return false
|
||||
}
|
||||
if (name.indexOf("大学") != -1 && !name.endsWith("大学")) {
|
||||
if (name.indexOf("校区") == -1 && name.indexOf("幼儿园") == -1 && name.indexOf("小学") == -1 && name.indexOf("中学") == -1
|
||||
&& !name.endsWith("医院")) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
if (name.indexOf("学院") != -1 && !name.endsWith("学院")) {
|
||||
return false
|
||||
}
|
||||
if (name.indexOf("分院") != -1 && !name.endsWith("分院")) {
|
||||
return false
|
||||
}
|
||||
if (name.indexOf("学校") != -1 && !name.endsWith("学校")) {
|
||||
if (name.indexOf("校区") == -1 && name.indexOf("分校") == -1) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
if (name.indexOf("中学") != -1 && !name.endsWith("中学")) {
|
||||
if (name.indexOf("校区") == -1) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
if (name.indexOf("小学") != -1 && !name.endsWith("小学")) {
|
||||
if (name.indexOf("校区") == -1) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
// 中梁山地铁站1A口
|
||||
if (obj.name.indexOf("地铁站") != -1 && obj.name.indexOf("口") != -1) {
|
||||
def nameTemp = obj.name.substring(0, obj.name.indexOf("地铁站") + 3)
|
||||
def key = nameTemp
|
||||
Map<String, Integer> child = new HashMap()
|
||||
if (map.get(key) != null) {
|
||||
int distance = Double.valueOf(obj.distance) + map.get(key).get("distance")
|
||||
int count1 = map.get(key).get("count")
|
||||
child.put("distance", distance)
|
||||
child.put("count", count1)
|
||||
map.put(key, child)
|
||||
} else {
|
||||
child.put("distance", Double.valueOf(obj.distance))
|
||||
child.put("count", 1)
|
||||
map.put(key, child)
|
||||
}
|
||||
} else {
|
||||
ret.add(obj.name + ":" + obj.distance + "米")
|
||||
}
|
||||
}
|
||||
|
||||
static printUnMatched(def unMatchedMap) {
|
||||
def ret = []
|
||||
ret << ['IDB','城市名称','区域编码','小区名称','楼栋号', '经度', '纬度']
|
||||
|
||||
for (def r :unMatchedMap.values()) {
|
||||
ret << r;
|
||||
}
|
||||
|
||||
com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx", ret, "周边配套");
|
||||
}
|
||||
|
||||
static String formatType(String typeStr) {
|
||||
def types = []
|
||||
if (typeStr.indexOf("|") > -1) {
|
||||
types = typeStr.split("\\|")
|
||||
} else {
|
||||
types =[typeStr]
|
||||
}
|
||||
for (def type : types) {
|
||||
if (type.startsWith("09")) {
|
||||
return "医院";
|
||||
} else if (type.startsWith('1505')) {
|
||||
return "地铁";
|
||||
} else if (type.startsWith('1507')) {
|
||||
return "公交";
|
||||
} else if (type.startsWith("16")) {
|
||||
return "银行";
|
||||
} else if (type.startsWith("14")) {
|
||||
return "学校";
|
||||
} else if (type.startsWith("06")) {
|
||||
return "商场"
|
||||
}
|
||||
}
|
||||
return "未知"
|
||||
}
|
||||
}
|
||||
|
||||
+61
@@ -0,0 +1,61 @@
|
||||
package com.fangjia.script.temp.shenjian.cqrcb
|
||||
|
||||
import com.fangjia.script.lixue.util.DBUtils
|
||||
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBObject
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
|
||||
/**
|
||||
* 通过匹配出的小区核对经纬度
|
||||
*
|
||||
* @author shenjian
|
||||
* @since 2021/5/19
|
||||
*/
|
||||
class CheckLatLngByDistrict {
|
||||
|
||||
static void main(String[] args) {
|
||||
List<List<Object>> data = ExcelUtils.read("F:\\数据\\重庆农商行\\测试数据汇总_2021.xlsx", 9)
|
||||
for (List<Object> districtT : data) {
|
||||
final List<Object> district = districtT
|
||||
if (district.size() >24 &&district.get(22) != null && district.get(23) != null && district.get(24) != null) {
|
||||
DBObject query = new BasicDBObject()
|
||||
query.put("city", district.get(22))
|
||||
query.put("region", district.get(23))
|
||||
query.put("cat", "district")
|
||||
query.put("visible", 0)
|
||||
query.put("name", district.get(24))
|
||||
if (district.get(20) != null && district.get(21) != null) {
|
||||
def originLng = district.get(20)
|
||||
def originLat = district.get(21)
|
||||
DBUtils.seaweed.find(query, new BasicDBObject("lng", 1).append("lat", 1)).limit(1).each { obj ->
|
||||
if (obj.lng && obj.lat) {
|
||||
if (originLng != obj.lng || originLat != obj.lat) {
|
||||
def nowLat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lat / 10000000000d);
|
||||
def nowLng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lng / 10000000000d);
|
||||
originLng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, originLng / 10000000000d);
|
||||
originLat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, originLat / 10000000000d);
|
||||
|
||||
double distance = GaoDeUtils.getDistance(nowLat, nowLng, originLat,originLng)
|
||||
println(district.get(24) + " 经度纬度不同" + nowLng + "," + nowLat + " 原始" + originLng + "," + originLat + " 距离" + distance + "米" )
|
||||
}
|
||||
} else {
|
||||
println(district.get(24) + " 不存在经纬度")
|
||||
}
|
||||
}
|
||||
} else {
|
||||
DBUtils.seaweed.find(query, new BasicDBObject("lng", 1).append("lat", 1)).limit(1).each { obj ->
|
||||
if (obj.lng && obj.lat) {
|
||||
|
||||
} else {
|
||||
println(district.get(24) + " 不存在经纬度222222222")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+2
-2
@@ -85,8 +85,8 @@ class SeaweedExport {
|
||||
filter.put("lat", 1)
|
||||
|
||||
DbUtils.seaweed.find(query).limit(1).each { obj ->
|
||||
ret[0] = obj.developer
|
||||
ret[1] = obj.estate_company
|
||||
ret[0] = obj.estate_company
|
||||
ret[1] = obj.developer
|
||||
ret[2] = SeaweedUtils.formatCubagePercent(obj.cubage_percent)
|
||||
ret[3] = SeaweedUtils.formatVirescencePercent(obj.virescence_percent)
|
||||
ret[4] = SeaweedUtils.formatAlias(obj.alias)
|
||||
|
||||
+7
-1
@@ -45,7 +45,7 @@ class GaoDeUtils {
|
||||
}};
|
||||
|
||||
/**
|
||||
*
|
||||
* 查找周边配套
|
||||
* @param city
|
||||
* @param kw
|
||||
* @param lng
|
||||
@@ -87,10 +87,16 @@ class GaoDeUtils {
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取经纬度之间距离
|
||||
*/
|
||||
static def getDistance(double Lat, double Lng, double Lat1, double Lng1) {
|
||||
return GeoHelper.GetDistance(Lat, Lng, Lat1, Lng1)
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取地址经纬度
|
||||
*/
|
||||
static def parserByGaode(def city, def address) {
|
||||
JSONObject json = GaoDeApiUtils.geo(city, address)
|
||||
if (json && json.status == "1") {
|
||||
|
||||
Reference in New Issue
Block a user