diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/RegionPoint.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/RegionPoint.groovy new file mode 100644 index 000000000..631de988e --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/RegionPoint.groovy @@ -0,0 +1,224 @@ +package com.fangjia.script.gj + +import com.fangjia.core.Constants +import com.fangjia.core.util.HttpRequest +import com.fangjia.core.util.Mongo3Utils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.FangJiaAreaMapUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.client.MongoCollection +import net.sf.json.JSONObject +import org.apache.commons.collections.CollectionUtils +import org.apache.commons.lang.math.RandomUtils +import org.apache.commons.lang3.StringUtils +import org.bson.Document + +import java.util.regex.Pattern + +/** + * 抓取区域边框 + */ +class RegionPoint { + + static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline") + static DBCollection GAODECITYCODECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "gaode_city_code") + + static main(args) { + def task = new RegionPoint(); + //河北省、山西省、辽宁省、吉林省、黑龙江省、江苏省、浙江省、 +// def provinces = ["安徽省","福建省","江西省","山东省","河南省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省"] + def provinces = ["内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"] + for(def p : provinces){ + task.crawlCityAndDistrictCode(p); + task.run(p); + } + + } + + def crawlCityAndDistrictCode(def province){ + crawlCity(province) + } + + def run(city){ + def cityCodes = findCityCode(city); + if(cityCodes){ + for(def cityMap : cityCodes){ + def codeMap = findAllDistrictCode(cityMap.city_code); + def index = 0 + for(def m : codeMap){ + index ++ + def ret = polygon(m.ad_code, "0", "all"); + if(!ret){ + println m.name +" " + m.ad_code + "无结果" + } + println index +" "+cityMap.name+" "+m.name+" "+ret.size() + if(ret.size()>0){ + saveData(cityMap.name, m.name, m.ad_code , ret); + } + } + } + } + } + + def crawlCity(def province){ + def ret = polygon(province, "2", "base"); + if(ret.size()>0){ + for(def map : ret){ + if(map.level == "province"){ + def cityList = map.districts + if(cityList && cityList.size()>0){ + for(def cityMap : cityList){ + if(cityMap.level == "city"){ + saveGaoDeCityRegionCode(province, cityMap.name, cityMap.citycode, cityMap.adcode, cityMap.center, cityMap.level); + saveGaoDeRegionCode(province, cityMap.districts); + } + } + } + } + } + } + } + + def saveGaoDeRegionCode(province, districts){ + if(districts){ + for(def dist : districts){ + if(dist.level=="district"){ + saveGaoDeCityRegionCode(province, dist.name, dist.citycode, dist.adcode, dist.center, dist.level); + } + } + } + } + + def findCityCode(city){ + Pattern pattern = Pattern.compile(city, Pattern.CASE_INSENSITIVE); + def q = null; + if(city.contains("省") || city.endsWith("自治区")){ + q = new BasicDBObject("province", pattern); + }else{ + q = new BasicDBObject("name", pattern); + } + q.put("level", "city") + def dbcursor = GAODECITYCODECOLL.find(q) + def rets = [] + for(def d : dbcursor){ + def city_code = d.get("city_code") + if(!rets.contains(city_code)){ + rets << d + } + } + return rets; + } + + private BasicDBObject getLikeStr(String findStr) { + Pattern pattern = Pattern.compile(findStr + ".*\$", Pattern.MULTILINE); + return new BasicDBObject("$regex", pattern); + } + + def findAllDistrictCode(def citycode){ + def q = new BasicDBObject() + q.put("city_code", citycode) + q.put("level", "district") + def dbcursor = GAODECITYCODECOLL.find(q) + def rets = [] + for(def d : dbcursor){ + rets << d + } + return rets; + } + + def saveGaoDeCityRegionCode(province, name, citycode, adcode, center, level){ + def q = new BasicDBObject() + q.put("city_code", citycode) + q.put("province", province) + q.put("level", level) + q.put("name", name) + if(adcode){ + q.put("ad_code", adcode) + } + + def o = GAODECITYCODECOLL.findOne(q) + if(!o){ + q.put("c_date", new Date()) + q.put("center", center) + GAODECITYCODECOLL.save(q); + }else{ + println name +" "+adcode+" 已存在" + } + } + + def saveData(city, region, code, ret){ + def q = new BasicDBObject() + q.put("code", code) + def dbo = POLYLINECOLL.findOne(q) + if(!dbo){ + def o = new BasicDBObject() + o.put("city", city) + o.put("region", region) + o.put("code", code) + o.put("center", ret.center.get(0)) + o.put("level", ret.level.get(0)) + o.put("polyline", ret.polyline.get(0)); + POLYLINECOLL.save(o); + } + } + + + public static def polygon(def keyword, def subdistrict, def extensions) { + String url = "https://restapi.amap.com/v3/config/district"; + + def params = [:] + params.put("keywords", keyword) + params.put("subdistrict",subdistrict) + params.put("extensions",extensions) + + String html = null; + def jo = null; + int errorCount = 0; + String status = "error" + def resultMap = [:] + try { + HttpRequest http = new HttpRequest(); + while (errorCount<5){//"2af0b8533415a612b98d59bb93d664b9" 公司key + def key = getRandomKey2() // + params.put("key", key) + html = http.get(url, params,"utf-8") + if (StringUtils.isNotEmpty(html)) { + jo = JSONObject.fromObject(html) + if(jo.infocode!="10000"){ + errorCount ++ + Thread.sleep(30*1000) +// println " 不可用Key: "+key + }else{ + errorCount = 100 + status = "normal" +// println " 可用Key: "+key + " "+html + } + } + } + } catch (Exception e) { + e.printStackTrace(); + } + + return jo.districts; + } + + static String getRandomKey2() { + def KeyList = [ + "2af0b8533415a612b98d59bb93d664b9", + "0246c57bbcf24aa246c207b3c99dfe41", + "042328b3f0fa5daf426802d379521d87", + "055d4f1b5b64a2bce18f4fe1f15c2639", + "d5484f5170502405d88bbaffe1b496f0", + "e74ccc4258079d2200199695110cceb1", + "f58629fc799fbc8537ca85046c6d44a5", + "291073f17ee0b963ccb1927ed92bf265", + "2a1fb75990f2b39f6221e50892dd3988" + ] + + def count = RandomUtils.nextInt(KeyList.size()) + return KeyList.get(count); + } +} +H \ No newline at end of file