添加小区围栏
This commit is contained in:
+286
@@ -0,0 +1,286 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/**
|
||||
* 添加高德地图,百度地图围栏
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class AddDistrictShapeBatch extends BaseBatch{
|
||||
|
||||
def fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
def mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
|
||||
|
||||
DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
|
||||
public static final DBCollection seaweed_fence_coll = MongoUtils.getCollectionByName("fangjia_base", "seaweed_fence", "mongo.db.fangjia.com")
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
static main(args) {
|
||||
runBatch(new AddDistrictShapeBatch(), args)
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def index = 1
|
||||
for(def c : citys){
|
||||
println c +" "+index
|
||||
index ++
|
||||
def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "estate_type2"])
|
||||
// baiduMatch(c, districts)
|
||||
gaodeMatch(c, districts)
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
def baiduMatch(city,districts){
|
||||
def data = fullCityMap.get(city)
|
||||
def fullCityName = null
|
||||
if(data){
|
||||
fullCityName = data.city
|
||||
}
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
|
||||
if(fullCityName){
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def d = districts[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def baiduId = baiduProcess(d, fullCityName)
|
||||
if(baiduId){
|
||||
if(!rets.contains(baiduId)){
|
||||
rets << baiduId
|
||||
}else{
|
||||
println "百度围栏 "+baiduId +" 被:["+d.city+" "+d.region+" "+d.name+" ]再次匹配,查看原始小区是否重复或别名相同"
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}else{
|
||||
println city +" 找不到相应的全称"
|
||||
}
|
||||
latch.await()
|
||||
}
|
||||
|
||||
|
||||
def baiduProcess(d, fullCityName){
|
||||
def alias = []
|
||||
alias << d.name
|
||||
if(d.alias){
|
||||
alias.addAll(d.alias)
|
||||
}
|
||||
|
||||
def distData = mcaDistMap.get(d.city+"#"+d.region)
|
||||
def fullDistName = null
|
||||
if(distData){
|
||||
fullDistName = distData.district
|
||||
}
|
||||
|
||||
def fenceMap = null
|
||||
for(def name : alias){
|
||||
fenceMap = findBaiduDatas(fullCityName, fullDistName, name)
|
||||
if(!fenceMap){
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if(fenceMap){
|
||||
def o = new BasicDBObject()
|
||||
o.put("city", d.city)
|
||||
o.put("district", d.region)
|
||||
o.put("name", d.name)
|
||||
o.put("source", "百度地图")
|
||||
o.put("c_date", new Date())
|
||||
o.put("data_id", fenceMap.data_id)
|
||||
o.put("comm_id", d._id.toString())
|
||||
def resultMap = [:]
|
||||
resultMap.coordinates = fenceMap.coordinatePairs
|
||||
// if(fenceMap.coordinatePairs.size()>1) {
|
||||
// resultMap.type = "MultiPolygon"
|
||||
// }else{
|
||||
resultMap.type = "polygon"
|
||||
// }
|
||||
o.put("shape", resultMap)
|
||||
|
||||
def q = new BasicDBObject("comm_id", d._id.toString())
|
||||
seaweed_fence_coll.update(q, new BasicDBObject('$set', o),true,false)
|
||||
// seaweed_fence_coll.save(o)
|
||||
println "[save] 围栏 ( "+d.city +" "+d.region +" "+d.name+" )"
|
||||
return fenceMap.data_id
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def findBaiduDatas(city, district, name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("district", district)
|
||||
q.put("name", name)
|
||||
q.put('std_tag', '房地产;住宅区')
|
||||
q.put("guoke_geo", new BasicDBObject('$ne', null))
|
||||
def dbcursor = BAIDU_POI_COLL.find(q)
|
||||
def map = [:]
|
||||
println dbcursor.count()
|
||||
for(def d : dbcursor){
|
||||
def arrs = d.gcj02_guoke_geo.split("\\|")
|
||||
|
||||
def coordinatesStr = arrs[2].toString().substring(2, arrs[2].toString().size()-1 )
|
||||
// 将字符串分割成列表
|
||||
def coordinatePairs = coordinatesStr.split(',')
|
||||
def coordinatesList = []
|
||||
for (int i = 0; i < coordinatePairs.size(); i += 2) {
|
||||
// 将字符串坐标转换为浮点数
|
||||
double longitude = coordinatePairs[i] as double
|
||||
double latitude = coordinatePairs[i + 1] as double
|
||||
|
||||
// 将坐标点作为一个元组添加到集合中
|
||||
coordinatesList << [longitude, latitude]
|
||||
}
|
||||
if(coordinatePairs){
|
||||
map.data_id = d._id
|
||||
map.coordinatePairs = coordinatesList
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
|
||||
def gaodeMatch(city,districts){
|
||||
def data = fullCityMap.get(city)
|
||||
def fullCityName = null
|
||||
if(data){
|
||||
fullCityName = data.city
|
||||
}
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
|
||||
if(fullCityName){
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def d = districts[i]
|
||||
def fence = findSeaweedFenceExists(d._id.toString())
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
if(!fence){
|
||||
def baiduId = gaodeProcess(d, fullCityName)
|
||||
if(baiduId){
|
||||
if(!rets.contains(baiduId)){
|
||||
rets << baiduId
|
||||
}else{
|
||||
println "高德围栏 "+baiduId +" 被:["+d.city+" "+d.region+" "+d.name+" ]再次匹配,查看原始小区是否重复或别名相同"
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}else{
|
||||
println city +" 找不到相应的全称"
|
||||
}
|
||||
latch.await()
|
||||
}
|
||||
|
||||
|
||||
def gaodeProcess(d, fullCityName){
|
||||
def alias = []
|
||||
alias << d.name
|
||||
if(d.alias){
|
||||
alias.addAll(d.alias)
|
||||
}
|
||||
|
||||
def distData = mcaDistMap.get(d.city+"#"+d.region)
|
||||
def fullDistName = null
|
||||
if(distData){
|
||||
fullDistName = distData.district
|
||||
}
|
||||
|
||||
def fenceMap = null
|
||||
for(def name : alias){
|
||||
fenceMap = findGaoDeDatas(fullCityName, fullDistName, name)
|
||||
if(!fenceMap){
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if(fenceMap){
|
||||
def o = new BasicDBObject()
|
||||
o.put("city", d.city)
|
||||
o.put("district", d.region)
|
||||
o.put("name", d.name)
|
||||
o.put("source", "高德地图")
|
||||
o.put("c_date", new Date())
|
||||
o.put("data_id", fenceMap.id)
|
||||
o.put("comm_id", d._id.toString())
|
||||
def resultMap = [:]
|
||||
|
||||
def shape = fenceMap.shape
|
||||
def points_x = fenceMap.shape.polygon.x
|
||||
def points_y = fenceMap.shape.polygon.y
|
||||
def list = []
|
||||
for(def i=0;i<points_x.size();i++){
|
||||
list << [points_x[i], points_y[i]]
|
||||
}
|
||||
resultMap.coordinates = list
|
||||
resultMap.type = "polygon"
|
||||
o.put("shape", resultMap)
|
||||
|
||||
def q = new BasicDBObject("comm_id", d._id.toString())
|
||||
seaweed_fence_coll.update(q, new BasicDBObject('$set', o),true,false)
|
||||
// seaweed_fence_coll.save(o)
|
||||
println "[save] 高德围栏 ( "+d.city +" "+d.region +" "+d.name+" )"
|
||||
return fenceMap.data_id
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def findGaoDeDatas(city, region, name){
|
||||
String sql = "select * from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}')"
|
||||
def obj = null
|
||||
ElasticUtils.executeSqlQuery(sql,{ d->
|
||||
def newName = d.name.replace("-","").replace("·","")
|
||||
if(newName == name && d.type.contains("住宅") && d.shape){
|
||||
obj = d
|
||||
}
|
||||
})
|
||||
return obj;
|
||||
}
|
||||
def findSeaweedFenceExists(commid){
|
||||
def q = new BasicDBObject()
|
||||
q.put("comm_id", commid)
|
||||
return seaweed_fence_coll.findOne(q)
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user