区域修改
This commit is contained in:
+82
-15
@@ -5,6 +5,7 @@ import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.lang.math.RandomUtils
|
||||
@@ -17,8 +18,9 @@ import java.util.regex.Pattern
|
||||
*/
|
||||
class RegionPointCrawl {
|
||||
|
||||
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
|
||||
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline_2025")
|
||||
static DBCollection GAODECITYCODECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "gaode_city_code")
|
||||
static DBCollection OLD_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
|
||||
|
||||
static main(args) {
|
||||
def task = new RegionPointCrawl();
|
||||
@@ -26,16 +28,19 @@ class RegionPointCrawl {
|
||||
//4个直辖市 北京市、上海市、天津市、重庆市
|
||||
//5个自治区 "内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"
|
||||
// def provinces = ["河北省", "山西省","吉林省","辽宁省", "黑龙江省", "陕西省", "甘肃省", "青海省", "山东省", "福建省", "浙江省", "台湾省", "河南省", "湖北省", "湖南省", "江西省", "江苏省", "安徽省", "广东省", "海南省", "四川省", "贵州省", "云南省"]
|
||||
def provinces = ["甘肃省"]
|
||||
def index = 0
|
||||
for(def p : provinces){
|
||||
index ++
|
||||
println index +"/"+provinces.size() +" "+p
|
||||
task.crawlCityAndDistrictCode(p);
|
||||
println "城市 区域 code 完成"
|
||||
task.run(p);
|
||||
println "区域围栏 完成"
|
||||
}
|
||||
//已跑:甘肃省,河北省,山西省,"吉林省","辽宁省", "黑龙江省", "陕西省", "青海省", "山东省", "福建省", "浙江省", "台湾省", "河南省", "湖北省", "湖南省", "江西省", "江苏省", "安徽省", "广东省", "海南省", "四川省", "贵州省","云南省"
|
||||
// def provinces = ["北京市","上海市","天津市","重庆市" ]
|
||||
// def index = 0
|
||||
// for(def p : provinces){
|
||||
// index ++
|
||||
// println index +"/"+provinces.size() +" "+p
|
||||
// task.crawlCityAndDistrictCode(p);
|
||||
// println "城市 区域 code 完成"
|
||||
// task.run(p);
|
||||
// println "区域围栏 完成"
|
||||
// }
|
||||
|
||||
task.statSame()
|
||||
|
||||
}
|
||||
|
||||
@@ -47,7 +52,10 @@ class RegionPointCrawl {
|
||||
def cityCodes = findCityCode(pro);
|
||||
if(cityCodes){
|
||||
for(def cityMap : cityCodes){
|
||||
def codeMap = findAllDistrictCode(cityMap.city_code);
|
||||
def codeMap = findAllDistrictCode(cityMap.city_code, "district");
|
||||
// if(!codeMap){
|
||||
// codeMap = findAllDistrictCode(cityMap.city_code, "street");
|
||||
// }
|
||||
def index = 0
|
||||
for(def m : codeMap){
|
||||
index ++
|
||||
@@ -93,7 +101,7 @@ class RegionPointCrawl {
|
||||
def saveGaoDeRegionCode(province, cityName,districts){
|
||||
if(districts){
|
||||
for(def dist : districts){
|
||||
if(dist.level=="district" || cityName=="东莞市"){
|
||||
if(dist.level=="district" || dist.level=="street" ){
|
||||
saveGaoDeCityRegionCode(province, dist.name, dist.citycode, dist.adcode, dist.center, dist.level);
|
||||
}
|
||||
}
|
||||
@@ -125,10 +133,10 @@ class RegionPointCrawl {
|
||||
return new BasicDBObject("$regex", pattern);
|
||||
}
|
||||
|
||||
def findAllDistrictCode(def citycode){
|
||||
def findAllDistrictCode(def citycode, def level){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city_code", citycode)
|
||||
q.put("level", "district")
|
||||
q.put("level", level)
|
||||
def dbcursor = GAODECITYCODECOLL.find(q)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
@@ -149,6 +157,7 @@ class RegionPointCrawl {
|
||||
|
||||
def o = GAODECITYCODECOLL.findOne(q)
|
||||
if(!o){
|
||||
println name +" "+adcode+" 新增"
|
||||
q.put("c_date", new Date())
|
||||
q.put("center", center)
|
||||
GAODECITYCODECOLL.save(q);
|
||||
@@ -233,4 +242,62 @@ class RegionPointCrawl {
|
||||
def count = RandomUtils.nextInt(KeyList.size())
|
||||
return KeyList.get(count);
|
||||
}
|
||||
|
||||
def statSame(){
|
||||
def dbcursor = POLYLINECOLL.find(new BasicDBObject()).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
|
||||
def index = 0
|
||||
for(def d : dbcursor){
|
||||
index++
|
||||
def q = new BasicDBObject()
|
||||
q.put("city",d.city)
|
||||
q.put("region", d.region)
|
||||
def obj = OLD_POLYLINECOLL.findOne(q)
|
||||
if(obj){
|
||||
if(obj.polyline!=d.polyline){
|
||||
println index +" "+ d.city +" "+d.region+" 2次区域不同"
|
||||
}
|
||||
}else{
|
||||
println index +" "+ d.city +" "+d.region+" 没有此区域"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def regionPolyline(){
|
||||
def dbcursor = POLYLINECOLL.find(new BasicDBObject("level","district"))
|
||||
def map = [:]
|
||||
for(def d : dbcursor){
|
||||
def resultMap = [:]
|
||||
resultMap.type = "polygon"
|
||||
def arrs = d.polyline.split("\\|")
|
||||
def rets = []
|
||||
for(def s : arrs){
|
||||
def points = s.split(";")
|
||||
def pointList = []
|
||||
for(def pointStr : points){
|
||||
def pointArr = pointStr.split(",")
|
||||
pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()]
|
||||
}
|
||||
rets << pointList
|
||||
}
|
||||
resultMap.coordinates = rets
|
||||
if(rets.size()>1) {
|
||||
resultMap.type = "MultiPolygon"
|
||||
}
|
||||
println d.city+" "+d.region+" "+rets.size()
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("shape", resultMap)
|
||||
def center = d.center
|
||||
if(center){
|
||||
def pointArrs = center.split(",")
|
||||
udata.put("location", ["lon":pointArrs[0].toString().toDouble(), "lat":pointArrs[1].toString().toDouble()])
|
||||
}
|
||||
|
||||
def fullCity = fullCityMap.get(d.city)
|
||||
map.put(fullCity.city+"|@|"+d.region, udata)
|
||||
}
|
||||
return map;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user