区域修改

This commit is contained in:
2025-04-08 14:48:29 +08:00
parent 780f1dccb1
commit 0a99a465d0
@@ -5,6 +5,7 @@ import com.fangjia.core.RegionData
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.apache.commons.lang.math.RandomUtils
@@ -17,8 +18,9 @@ import java.util.regex.Pattern
*/
class RegionPointCrawl {
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline_2025")
static DBCollection GAODECITYCODECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "gaode_city_code")
static DBCollection OLD_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
static main(args) {
def task = new RegionPointCrawl();
@@ -26,16 +28,19 @@ class RegionPointCrawl {
//4个直辖市 北京市、上海市、天津市、重庆市
//5个自治区 "内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"
// def provinces = ["河北省", "山西省","吉林省","辽宁省", "黑龙江省", "陕西省", "甘肃省", "青海省", "山东省", "福建省", "浙江省", "台湾省", "河南省", "湖北省", "湖南省", "江西省", "江苏省", "安徽省", "广东省", "海南省", "四川省", "贵州省", "云南省"]
def provinces = ["甘肃省"]
def index = 0
for(def p : provinces){
index ++
println index +"/"+provinces.size() +" "+p
task.crawlCityAndDistrictCode(p);
println "城市 区域 code 完成"
task.run(p);
println "区域围栏 完成"
}
//已跑:甘肃省,河北省,山西省,"吉林省","辽宁省", "黑龙江省", "陕西省", "青海省", "山东省", "福建省", "浙江省", "台湾省", "河南省", "湖北省", "湖南省", "江西省", "江苏省", "安徽省", "广东省", "海南省", "四川省", "贵州省","云南省"
// def provinces = ["北京市","上海市","天津市","重庆市" ]
// def index = 0
// for(def p : provinces){
// index ++
// println index +"/"+provinces.size() +" "+p
// task.crawlCityAndDistrictCode(p);
// println "城市 区域 code 完成"
// task.run(p);
// println "区域围栏 完成"
// }
task.statSame()
}
@@ -47,7 +52,10 @@ class RegionPointCrawl {
def cityCodes = findCityCode(pro);
if(cityCodes){
for(def cityMap : cityCodes){
def codeMap = findAllDistrictCode(cityMap.city_code);
def codeMap = findAllDistrictCode(cityMap.city_code, "district");
// if(!codeMap){
// codeMap = findAllDistrictCode(cityMap.city_code, "street");
// }
def index = 0
for(def m : codeMap){
index ++
@@ -93,7 +101,7 @@ class RegionPointCrawl {
def saveGaoDeRegionCode(province, cityName,districts){
if(districts){
for(def dist : districts){
if(dist.level=="district" || cityName=="东莞市"){
if(dist.level=="district" || dist.level=="street" ){
saveGaoDeCityRegionCode(province, dist.name, dist.citycode, dist.adcode, dist.center, dist.level);
}
}
@@ -125,10 +133,10 @@ class RegionPointCrawl {
return new BasicDBObject("$regex", pattern);
}
def findAllDistrictCode(def citycode){
def findAllDistrictCode(def citycode, def level){
def q = new BasicDBObject()
q.put("city_code", citycode)
q.put("level", "district")
q.put("level", level)
def dbcursor = GAODECITYCODECOLL.find(q)
def rets = []
for(def d : dbcursor){
@@ -149,6 +157,7 @@ class RegionPointCrawl {
def o = GAODECITYCODECOLL.findOne(q)
if(!o){
println name +" "+adcode+" 新增"
q.put("c_date", new Date())
q.put("center", center)
GAODECITYCODECOLL.save(q);
@@ -233,4 +242,62 @@ class RegionPointCrawl {
def count = RandomUtils.nextInt(KeyList.size())
return KeyList.get(count);
}
def statSame(){
def dbcursor = POLYLINECOLL.find(new BasicDBObject()).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def index = 0
for(def d : dbcursor){
index++
def q = new BasicDBObject()
q.put("city",d.city)
q.put("region", d.region)
def obj = OLD_POLYLINECOLL.findOne(q)
if(obj){
if(obj.polyline!=d.polyline){
println index +" "+ d.city +" "+d.region+" 2次区域不同"
}
}else{
println index +" "+ d.city +" "+d.region+" 没有此区域"
}
}
}
def regionPolyline(){
def dbcursor = POLYLINECOLL.find(new BasicDBObject("level","district"))
def map = [:]
for(def d : dbcursor){
def resultMap = [:]
resultMap.type = "polygon"
def arrs = d.polyline.split("\\|")
def rets = []
for(def s : arrs){
def points = s.split(";")
def pointList = []
for(def pointStr : points){
def pointArr = pointStr.split(",")
pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()]
}
rets << pointList
}
resultMap.coordinates = rets
if(rets.size()>1) {
resultMap.type = "MultiPolygon"
}
println d.city+" "+d.region+" "+rets.size()
def udata = new BasicDBObject()
udata.put("shape", resultMap)
def center = d.center
if(center){
def pointArrs = center.split(",")
udata.put("location", ["lon":pointArrs[0].toString().toDouble(), "lat":pointArrs[1].toString().toDouble()])
}
def fullCity = fullCityMap.get(d.city)
map.put(fullCity.city+"|@|"+d.region, udata)
}
return map;
}
}