add
This commit is contained in:
@@ -0,0 +1,224 @@
|
||||
package com.fangjia.script.gj
|
||||
|
||||
import com.fangjia.core.Constants
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.core.util.Mongo3Utils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.FangJiaAreaMapUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.client.MongoCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.collections.CollectionUtils
|
||||
import org.apache.commons.lang.math.RandomUtils
|
||||
import org.apache.commons.lang3.StringUtils
|
||||
import org.bson.Document
|
||||
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/**
|
||||
* 抓取区域边框
|
||||
*/
|
||||
class RegionPoint {
|
||||
|
||||
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
|
||||
static DBCollection GAODECITYCODECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "gaode_city_code")
|
||||
|
||||
static main(args) {
|
||||
def task = new RegionPoint();
|
||||
//河北省、山西省、辽宁省、吉林省、黑龙江省、江苏省、浙江省、
|
||||
// def provinces = ["安徽省","福建省","江西省","山东省","河南省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省"]
|
||||
def provinces = ["内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"]
|
||||
for(def p : provinces){
|
||||
task.crawlCityAndDistrictCode(p);
|
||||
task.run(p);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def crawlCityAndDistrictCode(def province){
|
||||
crawlCity(province)
|
||||
}
|
||||
|
||||
def run(city){
|
||||
def cityCodes = findCityCode(city);
|
||||
if(cityCodes){
|
||||
for(def cityMap : cityCodes){
|
||||
def codeMap = findAllDistrictCode(cityMap.city_code);
|
||||
def index = 0
|
||||
for(def m : codeMap){
|
||||
index ++
|
||||
def ret = polygon(m.ad_code, "0", "all");
|
||||
if(!ret){
|
||||
println m.name +" " + m.ad_code + "无结果"
|
||||
}
|
||||
println index +" "+cityMap.name+" "+m.name+" "+ret.size()
|
||||
if(ret.size()>0){
|
||||
saveData(cityMap.name, m.name, m.ad_code , ret);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def crawlCity(def province){
|
||||
def ret = polygon(province, "2", "base");
|
||||
if(ret.size()>0){
|
||||
for(def map : ret){
|
||||
if(map.level == "province"){
|
||||
def cityList = map.districts
|
||||
if(cityList && cityList.size()>0){
|
||||
for(def cityMap : cityList){
|
||||
if(cityMap.level == "city"){
|
||||
saveGaoDeCityRegionCode(province, cityMap.name, cityMap.citycode, cityMap.adcode, cityMap.center, cityMap.level);
|
||||
saveGaoDeRegionCode(province, cityMap.districts);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def saveGaoDeRegionCode(province, districts){
|
||||
if(districts){
|
||||
for(def dist : districts){
|
||||
if(dist.level=="district"){
|
||||
saveGaoDeCityRegionCode(province, dist.name, dist.citycode, dist.adcode, dist.center, dist.level);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def findCityCode(city){
|
||||
Pattern pattern = Pattern.compile(city, Pattern.CASE_INSENSITIVE);
|
||||
def q = null;
|
||||
if(city.contains("省") || city.endsWith("自治区")){
|
||||
q = new BasicDBObject("province", pattern);
|
||||
}else{
|
||||
q = new BasicDBObject("name", pattern);
|
||||
}
|
||||
q.put("level", "city")
|
||||
def dbcursor = GAODECITYCODECOLL.find(q)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
def city_code = d.get("city_code")
|
||||
if(!rets.contains(city_code)){
|
||||
rets << d
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
private BasicDBObject getLikeStr(String findStr) {
|
||||
Pattern pattern = Pattern.compile(findStr + ".*\$", Pattern.MULTILINE);
|
||||
return new BasicDBObject("$regex", pattern);
|
||||
}
|
||||
|
||||
def findAllDistrictCode(def citycode){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city_code", citycode)
|
||||
q.put("level", "district")
|
||||
def dbcursor = GAODECITYCODECOLL.find(q)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def saveGaoDeCityRegionCode(province, name, citycode, adcode, center, level){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city_code", citycode)
|
||||
q.put("province", province)
|
||||
q.put("level", level)
|
||||
q.put("name", name)
|
||||
if(adcode){
|
||||
q.put("ad_code", adcode)
|
||||
}
|
||||
|
||||
def o = GAODECITYCODECOLL.findOne(q)
|
||||
if(!o){
|
||||
q.put("c_date", new Date())
|
||||
q.put("center", center)
|
||||
GAODECITYCODECOLL.save(q);
|
||||
}else{
|
||||
println name +" "+adcode+" 已存在"
|
||||
}
|
||||
}
|
||||
|
||||
def saveData(city, region, code, ret){
|
||||
def q = new BasicDBObject()
|
||||
q.put("code", code)
|
||||
def dbo = POLYLINECOLL.findOne(q)
|
||||
if(!dbo){
|
||||
def o = new BasicDBObject()
|
||||
o.put("city", city)
|
||||
o.put("region", region)
|
||||
o.put("code", code)
|
||||
o.put("center", ret.center.get(0))
|
||||
o.put("level", ret.level.get(0))
|
||||
o.put("polyline", ret.polyline.get(0));
|
||||
POLYLINECOLL.save(o);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public static def polygon(def keyword, def subdistrict, def extensions) {
|
||||
String url = "https://restapi.amap.com/v3/config/district";
|
||||
|
||||
def params = [:]
|
||||
params.put("keywords", keyword)
|
||||
params.put("subdistrict",subdistrict)
|
||||
params.put("extensions",extensions)
|
||||
|
||||
String html = null;
|
||||
def jo = null;
|
||||
int errorCount = 0;
|
||||
String status = "error"
|
||||
def resultMap = [:]
|
||||
try {
|
||||
HttpRequest http = new HttpRequest();
|
||||
while (errorCount<5){//"2af0b8533415a612b98d59bb93d664b9" 公司key
|
||||
def key = getRandomKey2() //
|
||||
params.put("key", key)
|
||||
html = http.get(url, params,"utf-8")
|
||||
if (StringUtils.isNotEmpty(html)) {
|
||||
jo = JSONObject.fromObject(html)
|
||||
if(jo.infocode!="10000"){
|
||||
errorCount ++
|
||||
Thread.sleep(30*1000)
|
||||
// println " 不可用Key: "+key
|
||||
}else{
|
||||
errorCount = 100
|
||||
status = "normal"
|
||||
// println " 可用Key: "+key + " "+html
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
return jo.districts;
|
||||
}
|
||||
|
||||
static String getRandomKey2() {
|
||||
def KeyList = [
|
||||
"2af0b8533415a612b98d59bb93d664b9",
|
||||
"0246c57bbcf24aa246c207b3c99dfe41",
|
||||
"042328b3f0fa5daf426802d379521d87",
|
||||
"055d4f1b5b64a2bce18f4fe1f15c2639",
|
||||
"d5484f5170502405d88bbaffe1b496f0",
|
||||
"e74ccc4258079d2200199695110cceb1",
|
||||
"f58629fc799fbc8537ca85046c6d44a5",
|
||||
"291073f17ee0b963ccb1927ed92bf265",
|
||||
"2a1fb75990f2b39f6221e50892dd3988"
|
||||
]
|
||||
|
||||
def count = RandomUtils.nextInt(KeyList.size())
|
||||
return KeyList.get(count);
|
||||
}
|
||||
}
|
||||
H
|
||||
Reference in New Issue
Block a user