This commit is contained in:
2021-03-16 17:56:38 +08:00
parent ac85c4159d
commit 768fb924db
@@ -0,0 +1,224 @@
package com.fangjia.script.gj
import com.fangjia.core.Constants
import com.fangjia.core.util.HttpRequest
import com.fangjia.core.util.Mongo3Utils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.FangJiaAreaMapUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.client.MongoCollection
import net.sf.json.JSONObject
import org.apache.commons.collections.CollectionUtils
import org.apache.commons.lang.math.RandomUtils
import org.apache.commons.lang3.StringUtils
import org.bson.Document
import java.util.regex.Pattern
/**
* 抓取区域边框
*/
class RegionPoint {
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
static DBCollection GAODECITYCODECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "gaode_city_code")
static main(args) {
def task = new RegionPoint();
//河北省、山西省、辽宁省、吉林省、黑龙江省、江苏省、浙江省、
// def provinces = ["安徽省","福建省","江西省","山东省","河南省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省"]
def provinces = ["内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"]
for(def p : provinces){
task.crawlCityAndDistrictCode(p);
task.run(p);
}
}
def crawlCityAndDistrictCode(def province){
crawlCity(province)
}
def run(city){
def cityCodes = findCityCode(city);
if(cityCodes){
for(def cityMap : cityCodes){
def codeMap = findAllDistrictCode(cityMap.city_code);
def index = 0
for(def m : codeMap){
index ++
def ret = polygon(m.ad_code, "0", "all");
if(!ret){
println m.name +" " + m.ad_code + "无结果"
}
println index +" "+cityMap.name+" "+m.name+" "+ret.size()
if(ret.size()>0){
saveData(cityMap.name, m.name, m.ad_code , ret);
}
}
}
}
}
def crawlCity(def province){
def ret = polygon(province, "2", "base");
if(ret.size()>0){
for(def map : ret){
if(map.level == "province"){
def cityList = map.districts
if(cityList && cityList.size()>0){
for(def cityMap : cityList){
if(cityMap.level == "city"){
saveGaoDeCityRegionCode(province, cityMap.name, cityMap.citycode, cityMap.adcode, cityMap.center, cityMap.level);
saveGaoDeRegionCode(province, cityMap.districts);
}
}
}
}
}
}
}
def saveGaoDeRegionCode(province, districts){
if(districts){
for(def dist : districts){
if(dist.level=="district"){
saveGaoDeCityRegionCode(province, dist.name, dist.citycode, dist.adcode, dist.center, dist.level);
}
}
}
}
def findCityCode(city){
Pattern pattern = Pattern.compile(city, Pattern.CASE_INSENSITIVE);
def q = null;
if(city.contains("省") || city.endsWith("自治区")){
q = new BasicDBObject("province", pattern);
}else{
q = new BasicDBObject("name", pattern);
}
q.put("level", "city")
def dbcursor = GAODECITYCODECOLL.find(q)
def rets = []
for(def d : dbcursor){
def city_code = d.get("city_code")
if(!rets.contains(city_code)){
rets << d
}
}
return rets;
}
private BasicDBObject getLikeStr(String findStr) {
Pattern pattern = Pattern.compile(findStr + ".*\$", Pattern.MULTILINE);
return new BasicDBObject("$regex", pattern);
}
def findAllDistrictCode(def citycode){
def q = new BasicDBObject()
q.put("city_code", citycode)
q.put("level", "district")
def dbcursor = GAODECITYCODECOLL.find(q)
def rets = []
for(def d : dbcursor){
rets << d
}
return rets;
}
def saveGaoDeCityRegionCode(province, name, citycode, adcode, center, level){
def q = new BasicDBObject()
q.put("city_code", citycode)
q.put("province", province)
q.put("level", level)
q.put("name", name)
if(adcode){
q.put("ad_code", adcode)
}
def o = GAODECITYCODECOLL.findOne(q)
if(!o){
q.put("c_date", new Date())
q.put("center", center)
GAODECITYCODECOLL.save(q);
}else{
println name +" "+adcode+" 已存在"
}
}
def saveData(city, region, code, ret){
def q = new BasicDBObject()
q.put("code", code)
def dbo = POLYLINECOLL.findOne(q)
if(!dbo){
def o = new BasicDBObject()
o.put("city", city)
o.put("region", region)
o.put("code", code)
o.put("center", ret.center.get(0))
o.put("level", ret.level.get(0))
o.put("polyline", ret.polyline.get(0));
POLYLINECOLL.save(o);
}
}
public static def polygon(def keyword, def subdistrict, def extensions) {
String url = "https://restapi.amap.com/v3/config/district";
def params = [:]
params.put("keywords", keyword)
params.put("subdistrict",subdistrict)
params.put("extensions",extensions)
String html = null;
def jo = null;
int errorCount = 0;
String status = "error"
def resultMap = [:]
try {
HttpRequest http = new HttpRequest();
while (errorCount<5){//"2af0b8533415a612b98d59bb93d664b9" 公司key
def key = getRandomKey2() //
params.put("key", key)
html = http.get(url, params,"utf-8")
if (StringUtils.isNotEmpty(html)) {
jo = JSONObject.fromObject(html)
if(jo.infocode!="10000"){
errorCount ++
Thread.sleep(30*1000)
// println " 不可用Key: "+key
}else{
errorCount = 100
status = "normal"
// println " 可用Key: "+key + " "+html
}
}
}
} catch (Exception e) {
e.printStackTrace();
}
return jo.districts;
}
static String getRandomKey2() {
def KeyList = [
"2af0b8533415a612b98d59bb93d664b9",
"0246c57bbcf24aa246c207b3c99dfe41",
"042328b3f0fa5daf426802d379521d87",
"055d4f1b5b64a2bce18f4fe1f15c2639",
"d5484f5170502405d88bbaffe1b496f0",
"e74ccc4258079d2200199695110cceb1",
"f58629fc799fbc8537ca85046c6d44a5",
"291073f17ee0b963ccb1927ed92bf265",
"2a1fb75990f2b39f6221e50892dd3988"
]
def count = RandomUtils.nextInt(KeyList.size())
return KeyList.get(count);
}
}
H