重庆农商行导出

This commit is contained in:
沈健
2021-05-17 10:42:29 +08:00
parent cb2ee7ebda
commit 964b25baa2
10 changed files with 400 additions and 125 deletions
@@ -28,6 +28,7 @@ class AroundExportByNewWay {
static DBCollection poiConn = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017)
static def resultMap = new ConcurrentHashMap()
static def unMatchedMap = new ConcurrentHashMap()
static ExecutorService executorService = Executors.newFixedThreadPool(30)
static void main(String[] args) {
@@ -42,9 +43,6 @@ class AroundExportByNewWay {
try {
def id = district.get(0)
def district_code = district.get(2)
if (district_code == 'B100203') {
println(district_code)
}
def name = district.get(3)
List result = []
if (district.get(20) != null && district.get(21) != null) {
@@ -57,24 +55,28 @@ class AroundExportByNewWay {
q.put('status', 0)
q.put('lat', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLat).append(QueryOperators.GTE, mapLatLng.minLat));
q.put('lng', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLng).append(QueryOperators.GTE, mapLatLng.minLng));
Map<String, Map> map = new ConcurrentHashMap<>();
poiConn.find(q, new BasicDBObject("name": 1, "distance": 1, "type": 1, "typecode": 1, "lat": 1, "lng": 1)).batchSize(500).each {obj ->
double distance = Distance.GetDistance(lng, lat, obj.lng, obj.lat)
if (distance <= 3000) {
dealSpecial(obj, map, id, distance, result)
if (distance == 0) {
distance = 50
}
def ret = []
ret[0] = id
ret[1] = obj.name
ret[2] = formatType(obj.typecode)
ret[3] = distance.toInteger()
result << ret
}
}
for (String key : map.keySet()) {
int distance = map.get(key).get("distance")
int count1 = map.get(key).get("count")
def type = map.get(key).get("type")
def ret = []
ret[0] = id
ret[1] = key
ret[2] = type
ret[3] = (distance / count1).toInteger()
result << ret
}
} else {
def ret = []
ret[0] = id
ret[1] = district.get(1)
ret[2] = district_code
ret[3] = name
ret[4] = district.get(4)
unMatchedMap.put(id, ret)
}
resultMap.put(district_code + name, result)
println("当前处理" + count.incrementAndGet() )
@@ -86,100 +88,63 @@ class AroundExportByNewWay {
}
})
};
latch.await()
latch.await();
def headArr = ['IDB', '名称','类型', '距离']
List<String> head = new ArrayList<>()
for (String name: headArr) {
List<String> temp = new ArrayList<>()
temp.add(name)
head.add(temp)
}
Map<Integer, List> map = new HashedMap()
Integer key = 1;
def ret = []
for (List<Object> district : data) {
def district_code = district.get(2)
def name = district.get(3)
if (com.fangjia.script.coredata.util.CollectionUtils.isNotEmpty(resultMap.get(district_code + name))) {
ret.addAll(resultMap.get(district_code + name))
if (ret.size() > 600000) {
map.put(key++, ret)
ret = []
}
}
}
println(ret.size())
map.put(key++, ret)
//com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套.xlsx", ret, "周边配套");
FileUtils.batchCreateFile(ret, "F:\\数据\\重庆农商行\\周边配套.dat", headArr, "utf-8")
// EasyExcel.write("F:\\数据\\重庆农商行\\周边配套.xlsx").head(head).sheet("成交信息").doWrite(ret);
for (Integer k : map.keySet()) {
FileUtils.batchCreateFile(map.get(k), "F:\\数据\\重庆农商行\\周边配套"+ k + ".dat", headArr, "utf-8")
//EasyExcel.write("F:\\数据\\重庆农商行\\周边配套"+ + k + ".xlsx").head(head).sheet("周边配套").doWrite(map.get(k));
}
//printUnMatched(unMatchedMap)
executorService.shutdown()
}
static dealSpecial(DBObject obj, def map, def id, double distance, def result) {
def name = obj.name
if (name.indexOf("/") != -1) {
return false
static printUnMatched(def unMatchedMap) {
def ret = []
ret << ['IDB','城市名称','区域编码','小区名称','楼栋号', '经度', '纬度']
for (def r :unMatchedMap.values()) {
ret << r;
}
if (name == "1号楼") {
return false
}
if (name.indexOf("地铁口") == -1 && name.indexOf("公交站") == -1 && name.indexOf("支行") == -1) {
if (name.indexOf("栋") != -1) {
return false
}
if (name.indexOf("医院") != -1 && !name.endsWith("医院")) {
if (name.indexOf("分院") == -1) {
return false
}
}
if (name.indexOf("卫生中心") != -1 && !name.endsWith("卫生中心")) {
return false
}
if (name.indexOf("大学") != -1 && !name.endsWith("大学")) {
if (name.indexOf("校区") == -1 && name.indexOf("幼儿园") == -1 && name.indexOf("小学") == -1 && name.indexOf("中学") == -1
&& !name.endsWith("医院")) {
return false
}
}
if (name.indexOf("学院") != -1 && !name.endsWith("学院")) {
return false
}
if (name.indexOf("分院") != -1 && !name.endsWith("分院")) {
return false
}
if (name.indexOf("学校") != -1 && !name.endsWith("学校")) {
if (name.indexOf("校区") == -1 && name.indexOf("分校") == -1) {
return false
}
}
if (name.indexOf("中学") != -1 && !name.endsWith("中学")) {
if (name.indexOf("校区") == -1) {
return false
}
}
if (name.indexOf("小学") != -1 && !name.endsWith("小学")) {
if (name.indexOf("校区") == -1) {
return false
}
}
if (name.indexOf("(出入口)") != -1) {
return false
}
}
// 中梁山地铁站1A口
if (obj.name.indexOf("地铁站") != -1 && obj.name.indexOf("口") != -1) {
def nameTemp = obj.name.substring(0, obj.name.indexOf("地铁站") + 3)
def key = nameTemp
Map<String, Integer> child = new HashMap()
if (map.get(key) != null) {
distance + map.get(key).get("distance")
int count1 = map.get(key).get("count")
child.put("distance", distance)
child.put("count", count1)
child.put("type", obj.type)
map.put(key, child)
} else {
child.put("distance", distance)
child.put("type", obj.type)
child.put("count", 1)
map.put(key, child)
}
} else {
def ret = []
ret[0] = id
ret[1] = obj.name
ret[2] = obj.type
ret[3] = distance.toInteger()
result << ret
com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx", ret, "周边配套");
}
static String formatType(String type) {
if (type.startsWith("09")) {
return "医院";
} else if (type.startsWith('1505')) {
return "地铁";
} else if (type.startsWith('1507')) {
return "公交";
} else if (type.startsWith("16")) {
return "银行";
} else if (type.startsWith("14")) {
return "学校";
} else if (type.startsWith("06")) {
return "商场"
}
}
}
@@ -0,0 +1,106 @@
package com.fangjia.script.temp.shenjian.cqrcb
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import com.mongodb.QueryOperators
/**
* 处理未匹配的小区
*
* @author shenjian
* @since 2021/5/17
*/
class DealUnMatched {
static void main(String[] args) {
// getLatLngByName()
// getLatLngByNumber()
//findExceptionDistance()
// B114862
def lng = 107.736591
def lat = 29.872502
// 周围半径为3千米
def mapLatLng = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 500)
DBObject q = new BasicDBObject()
q.put('city', '重庆')
q.put('status', 0)
q.put('lat', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLat).append(QueryOperators.GTE, mapLatLng.minLat));
q.put('lng', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLng).append(QueryOperators.GTE, mapLatLng.minLng));
DBUtils.seaweed.find(q).batchSize(100).each {obj ->
println(obj)
}
}
static void findExceptionDistance() {
List<List<Object>> data = ExcelUtils.read("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx",0)
for (List<Object> district : data) {
if (district.size() > 12) {
def lng = java.lang.Double.valueOf(district.get(8))
def lat = java.lang.Double.valueOf(district.get(9))
def lng1 = java.lang.Double.valueOf(district.get(11))
def lat1 = java.lang.Double.valueOf(district.get(12))
double distance = GaoDeUtils.getDistance(lat, lng, lat1, lng1)
if (distance > 500) {
println("距离异常" + district)
district.add(distance)
district.add("距离异常")
}
}
}
data.add(0, new ArrayList<Object>())
ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx", data, "周边配套")
}
static void getLatLngByNumber() {
List<List<Object>> data = ExcelUtils.read("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx",1)
for (List<Object> districtT : data) {
final List<Object> district = districtT
def city = district.get(1)
def district_code = district.get(2)
def name = district.get(3)
def number = district.get(4)
//def region = SeaweedUtils.getRegionByCode(district_code)
if (district.size() <= 5) {
if (number != null && number.size() > 5) {
def map = GaoDeUtils.parserByGaode(city, number)
if (map != null) {
district.add(map.get("lng"))
district.add(map.get("lat"))
district.add("通过楼栋得出")
}
} else {
println(district_code + " " + name + " " + number)
}
}
}
ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx", data, "未匹配1")
}
static void getLatLngByName() {
List<List<Object>> data = ExcelUtils.read("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx", 0)
for (List<Object> districtT : data) {
final List<Object> district = districtT
def city = district.get(1)
def district_code = district.get(2)
def name = district.get(3)
def number = district.get(4)
//def region = SeaweedUtils.getRegionByCode(district_code)
def map = GaoDeUtils.parserByGaode(city, name)
if (map != null) {
district.add(map.get("lng"))
district.add(map.get("lat"))
district.add("通过小区得出")
}
}
ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx", data, "未匹配")
}
}
@@ -0,0 +1,53 @@
package com.fangjia.script.temp.shenjian.cqrcb
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
/**
* 导入中学小学信息
*
* @author shenjian
* @since 2021/5/17
*/
class ImportSchool {
static DBCollection poiConn = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017)
static void main(String[] args) {
List<List<Object>> data = ExcelUtils.read("F:\\数据\\重庆农商行\\重庆学校_0902.xlsx", 1)
List result = new ArrayList();
for (List<Object> school : data) {
try {
def city = school.get(0)
def region = school.get(1)
def name = school.get(2)
def address = school.get(4)
def type = school.get(6)
def lng = school.get(9)
def lat = school.get(10)
DBObject dbObject = new BasicDBObject()
dbObject.put("province", "重庆")
dbObject.put("city", city)
dbObject.put("region", region)
dbObject.put("name", name)
dbObject.put("address", address)
dbObject.put("type", type)
dbObject.put("typecode", "141200")
dbObject.put("lng", lng)
dbObject.put("lat", lat)
dbObject.put("status", 0)
dbObject.put("flag", "excel导入")
result.add(dbObject)
} catch(Exception ex) {
println(school)
ex.printStackTrace()
}
}
poiConn.insert(result)
}
}
@@ -1,13 +1,17 @@
package com.fangjia.script.temp.shenjian.cqrcb
import cn.emay.mina.util.ConcurrentHashSet
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
/**
* POI迁移
@@ -22,18 +26,20 @@ class MoveArroundTemp {
static ExecutorService executorService = Executors.newFixedThreadPool(30)
static void main(String[] args) {
List<DBObject> list = Collections.synchronizedList(new ArrayList<DBObject>())
List<String> ids = conn.distinct("id")
CountDownLatch latch = new CountDownLatch(ids.size())
for (String idT : ids) {
final String id = idT
Map<String, DBObject> map = new ConcurrentHashMap()
AtomicInteger count = new AtomicInteger()
int size = conn.count(new BasicDBObject("status", 1))
println(size)
CountDownLatch latch = new CountDownLatch(size)
conn.find(new BasicDBObject("status", 1)).batchSize(10000).each { objT ->
final def obj = objT
executorService.submit(new Runnable() {
@Override
void run() {
try {
DBObject dbObject = new BasicDBObject()
conn.find(new BasicDBObject("id", id)).limit(1).each { obj ->
dbObject.put("_id", id)
if (map.get(obj.id) == null && conn43.count(new BasicDBObject("_id", obj.id)) <= 0) {
DBObject dbObject = new BasicDBObject()
dbObject.put("_id", obj.id)
dbObject.put("name", obj.name)
dbObject.put("type", obj.type)
dbObject.put("typecode", obj.typecode)
@@ -44,21 +50,66 @@ class MoveArroundTemp {
dbObject.put("province", obj.pname)
dbObject.put("city", obj.cityname)
dbObject.put("region", obj.adname)
dbObject.put("status", 2)
dbObject.put("flag", "待清洗")
map.put(obj.id, dbObject)
}
list.add(dbObject)
} catch(Exception ex) {
ex.printStackTrace()
} finally {
latch.countDown()
println("当前记录数" + list.size())
println(count.incrementAndGet())
}
}
})
};
latch.await();
conn43.insert(list);
list.clear();
println("同步清理完成");
}
latch.await()
List ret = new ArrayList()
for (String key : map.keySet()) {
ret << map.get(key)
}
conn43.insert(ret)
executorService.shutdown()
}
static origin() {
// List<DBObject> list = Collections.synchronizedList(new ArrayList<DBObject>())
// List<String> ids = conn.distinct("id")
// CountDownLatch latch = new CountDownLatch(ids.size())
// for (String idT : ids) {
// final String id = idT
// executorService.submit(new Runnable() {
// @Override
// void run() {
// try {
// DBObject dbObject = new BasicDBObject()
// conn.find(new BasicDBObject("id", id)).limit(1).each { obj ->
// dbObject.put("_id", id)
// dbObject.put("name", obj.name)
// dbObject.put("type", obj.type)
// dbObject.put("typecode", obj.typecode)
// dbObject.put("address", obj.address)
// dbObject.put("lng", Double.valueOf(obj.location.split(",")[0]))
// dbObject.put("lat", Double.valueOf(obj.location.split(",")[1]))
// dbObject.put("tel", obj.tel)
// dbObject.put("province", obj.pname)
// dbObject.put("city", obj.cityname)
// dbObject.put("region", obj.adname)
// }
// list.add(dbObject)
// } catch(Exception ex) {
// ex.printStackTrace()
// } finally {
// latch.countDown()
// println("当前记录数" + list.size())
// }
// }
// })
// };
// latch.await();
// conn43.insert(list);
// list.clear();
// println("同步清理完成");
// executorService.shutdown()
}
}
@@ -44,10 +44,11 @@ class POIRule {
}
// 更新规则
// db.getCollection('gaode_address_poi').update({'name': /栋/, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含栋'}}, {multi:true,upsert:false});
//db.getCollection('gaode_address_poi').update({'name': /幢/, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含幢'}}, {multi:true,upsert:false});
// db.getCollection('gaode_address_poi').update({'name': /[ABCD]+栋/, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含栋'}}, {multi:true,upsert:false});
//db.getCollection('gaode_address_poi').update({'name': /[ABCD]+幢/, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含幢'}}, {multi:true,upsert:false});
//db.getCollection('gaode_address_poi').update({'name': /[ABCD]+馆/, 'typecode': {$in: ['060102', '060101']}, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含馆'}}, {multi:true,upsert:false});
//db.getCollection('gaode_address_poi').update({'name': /[ABCDE1234567890]+区/, 'typecode': {$nin: ['150700']}, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含区'}}, {multi:true,upsert:false});
//db.getCollection('gaode_address_poi').update({'name': /[ABCD]口/, 'typecode': {$nin: ['150700']}, 'status': 2}, {$set: {'status': NumberInt(1), 'flag': '包含入口'}}, {multi:true,upsert:false});
//db.getCollection('gaode_address_poi').update({'name': /入口/, 'typecode': {$nin: ['150700']}, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含入口'}}, {multi:true,upsert:false});
//db.getCollection('gaode_address_poi').update({'type': /医疗保健服务;专科医院;整形美容/}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false});
//db.getCollection('gaode_address_poi').update({'type': /科教文化服务;学校;职业技术学校/}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false});
@@ -63,4 +64,7 @@ class POIRule {
//db.getCollection('gaode_address_poi').update({'name': /智能柜/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false});
//db.getCollection('gaode_address_poi').update({'name': /摄影/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false});
//db.getCollection('gaode_address_poi').update({'typecode': '141204', 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false});
//db.getCollection('gaode_address_poi').update({'typecode': /1412/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false});
//
//db.getCollection('gaode_address_poi').update({'name': '西南大学'}, {$set: {'status': NumberInt(0)}}, {multi:true,upsert:false});
}
@@ -0,0 +1,55 @@
package com.fangjia.script.temp.shenjian.cqrcb
import com.fangjia.script.coredata.util.CollectionUtils
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
/**
* @author shenjian
* @since 2021/5/17
*/
class SavePoi {
static DBCollection conn = MongoUtils.getCollectionByName("seeddb_dev", "arround_temp", "192.168.0.235", 27018)
static DBCollection conn43 = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017)
//3km范围内地铁站、公交站、银行、医院、商场、学校等名称及距离信息
static def types = "150500|1507|090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211|1601|0601|141200|141201|141202|141203|141204"
static def radius = "3000"
static ExecutorService executorService = Executors.newFixedThreadPool(30)
static void main(String[] args) {
List<List<Object>> data = ExcelUtils.read("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx",0)
AtomicInteger count = new AtomicInteger()
for (List<Object> listT : data) {
final list = listT
executorService.execute(new Runnable() {
@Override
void run() {
if (list.size() >= 17) {
def city = list.get(1)
def lng = Double.valueOf(list.get(11))
def lat = Double.valueOf(list.get(12))
List arounds = GaoDeUtils.searchNearByPoi(city, types, lng, lat, radius, "")
if (CollectionUtils.isNotEmpty(arounds)) {
for (BasicDBObject obj : arounds) {
obj.put("status", 1)
obj.put("flag", "距离异常入库")
}
conn.insert(arounds)
println(count.incrementAndGet())
}
}
}
})
};
executorService.shutdown()
}
}
@@ -95,7 +95,8 @@ class SeaweedExport {
for (def a : obj.alias) {
temp.add(a)
}
ret[4] = temp.join(",")
String als = temp.join(",")
ret[4] = als.length() > 20 ? "" : als
}
ret[5] = obj.car_site_count
if (com.fangjia.script.datastore.FieldValidatorUtils.householdCountValidator(obj.household_count)
@@ -75,16 +75,5 @@ class DistanceDeal {
// }
}
static def parserByGaode(def city, def address) {
JSONObject json = GaoDeApiUtils.geo(city, address)
if (json && json.status == "1") {
def ret = json.geocodes
if (ret) {
def str = ret.get(0).location
def level = ret.get(0).level
def arrs = str.split(",")
return ["lng": arrs[0], "lat": arrs[1]]
}
}
}
}
@@ -1,7 +1,9 @@
package com.fangjia.script.temp.shenjian.util
import com.fangjia.core.util.DataUtil
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.HttpRequestUtil
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import net.sf.json.JSONObject
import java.util.concurrent.ConcurrentHashMap
@@ -84,4 +86,21 @@ class GaoDeUtils {
}
return result
}
static def getDistance(double Lat, double Lng, double Lat1, double Lng1) {
return GeoHelper.GetDistance(Lat, Lng, Lat1, Lng1)
}
static def parserByGaode(def city, def address) {
JSONObject json = GaoDeApiUtils.geo(city, address)
if (json && json.status == "1") {
def ret = json.geocodes
if (ret) {
def str = ret.get(0).location
def level = ret.get(0).level
def arrs = str.split(",")
return ["lng": arrs[0], "lat": arrs[1]]
}
}
}
}
@@ -0,0 +1,32 @@
package com.fangjia.script.temp.shenjian.util
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.ConcurrentHashMap
/**
* Ð¡Çø¹¤¾ßÀà
*
* @author shenjian
* @since 2021/5/17
*/
class SeaweedUtils {
static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017)
static def map = new ConcurrentHashMap()
static getRegionByCode(def district_code) {
def region;
if (map.get("district_code") != null) {
region = map.get("district_code")
} else {
cityMapConn.find(new BasicDBObject("district_code", district_code)).each { obj ->
region = obj.fangjia_district
map.put("district_code", region)
}
}
return region
}
}