diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/RegionPoint.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/RegionPoint.groovy index e80dd9c35..030658e63 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/RegionPoint.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/RegionPoint.groovy @@ -5,6 +5,7 @@ import com.fangjia.core.RegionData import com.fangjia.core.util.HttpRequest import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject +import com.mongodb.Bytes import com.mongodb.DBCollection import net.sf.json.JSONObject import org.apache.commons.lang.math.RandomUtils @@ -17,8 +18,9 @@ import java.util.regex.Pattern */ class RegionPointCrawl { - static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline") + static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline_2025") static DBCollection GAODECITYCODECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "gaode_city_code") + static DBCollection OLD_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline") static main(args) { def task = new RegionPointCrawl(); @@ -26,16 +28,19 @@ class RegionPointCrawl { //4个直辖市 北京市、上海市、天津市、重庆市 //5个自治区 "内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区" // def provinces = ["河北省", "山西省","吉林省","辽宁省", "黑龙江省", "陕西省", "甘肃省", "青海省", "山东省", "福建省", "浙江省", "台湾省", "河南省", "湖北省", "湖南省", "江西省", "江苏省", "安徽省", "广东省", "海南省", "四川省", "贵州省", "云南省"] - def provinces = ["甘肃省"] - def index = 0 - for(def p : provinces){ - index ++ - println index +"/"+provinces.size() +" "+p - task.crawlCityAndDistrictCode(p); - println "城市 区域 code 完成" - task.run(p); - println "区域围栏 完成" - } + //已跑:甘肃省,河北省,山西省,"吉林省","辽宁省", "黑龙江省", "陕西省", "青海省", "山东省", "福建省", "浙江省", "台湾省", "河南省", "湖北省", "湖南省", "江西省", "江苏省", "安徽省", "广东省", "海南省", "四川省", "贵州省","云南省" +// def provinces = ["北京市","上海市","天津市","重庆市" ] +// def index = 0 +// for(def p : provinces){ +// index ++ +// println index +"/"+provinces.size() +" "+p +// task.crawlCityAndDistrictCode(p); +// println "城市 区域 code 完成" +// task.run(p); +// println "区域围栏 完成" +// } + + task.statSame() } @@ -47,7 +52,10 @@ class RegionPointCrawl { def cityCodes = findCityCode(pro); if(cityCodes){ for(def cityMap : cityCodes){ - def codeMap = findAllDistrictCode(cityMap.city_code); + def codeMap = findAllDistrictCode(cityMap.city_code, "district"); +// if(!codeMap){ +// codeMap = findAllDistrictCode(cityMap.city_code, "street"); +// } def index = 0 for(def m : codeMap){ index ++ @@ -93,7 +101,7 @@ class RegionPointCrawl { def saveGaoDeRegionCode(province, cityName,districts){ if(districts){ for(def dist : districts){ - if(dist.level=="district" || cityName=="东莞市"){ + if(dist.level=="district" || dist.level=="street" ){ saveGaoDeCityRegionCode(province, dist.name, dist.citycode, dist.adcode, dist.center, dist.level); } } @@ -125,10 +133,10 @@ class RegionPointCrawl { return new BasicDBObject("$regex", pattern); } - def findAllDistrictCode(def citycode){ + def findAllDistrictCode(def citycode, def level){ def q = new BasicDBObject() q.put("city_code", citycode) - q.put("level", "district") + q.put("level", level) def dbcursor = GAODECITYCODECOLL.find(q) def rets = [] for(def d : dbcursor){ @@ -149,6 +157,7 @@ class RegionPointCrawl { def o = GAODECITYCODECOLL.findOne(q) if(!o){ + println name +" "+adcode+" 新增" q.put("c_date", new Date()) q.put("center", center) GAODECITYCODECOLL.save(q); @@ -233,4 +242,62 @@ class RegionPointCrawl { def count = RandomUtils.nextInt(KeyList.size()) return KeyList.get(count); } + + def statSame(){ + def dbcursor = POLYLINECOLL.find(new BasicDBObject()).setOptions(Bytes.QUERYOPTION_NOTIMEOUT) + + def index = 0 + for(def d : dbcursor){ + index++ + def q = new BasicDBObject() + q.put("city",d.city) + q.put("region", d.region) + def obj = OLD_POLYLINECOLL.findOne(q) + if(obj){ + if(obj.polyline!=d.polyline){ + println index +" "+ d.city +" "+d.region+" 2次区域不同" + } + }else{ + println index +" "+ d.city +" "+d.region+" 没有此区域" + } + } + } + + + def regionPolyline(){ + def dbcursor = POLYLINECOLL.find(new BasicDBObject("level","district")) + def map = [:] + for(def d : dbcursor){ + def resultMap = [:] + resultMap.type = "polygon" + def arrs = d.polyline.split("\\|") + def rets = [] + for(def s : arrs){ + def points = s.split(";") + def pointList = [] + for(def pointStr : points){ + def pointArr = pointStr.split(",") + pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()] + } + rets << pointList + } + resultMap.coordinates = rets + if(rets.size()>1) { + resultMap.type = "MultiPolygon" + } + println d.city+" "+d.region+" "+rets.size() + + def udata = new BasicDBObject() + udata.put("shape", resultMap) + def center = d.center + if(center){ + def pointArrs = center.split(",") + udata.put("location", ["lon":pointArrs[0].toString().toDouble(), "lat":pointArrs[1].toString().toDouble()]) + } + + def fullCity = fullCityMap.get(d.city) + map.put(fullCity.city+"|@|"+d.region, udata) + } + return map; + } }