add
This commit is contained in:
@@ -0,0 +1,224 @@
|
|||||||
|
package com.fangjia.script.gj
|
||||||
|
|
||||||
|
import com.fangjia.core.Constants
|
||||||
|
import com.fangjia.core.util.HttpRequest
|
||||||
|
import com.fangjia.core.util.Mongo3Utils
|
||||||
|
import com.fangjia.script.commData.CommonDatautils
|
||||||
|
import com.fangjia.script.util.FangJiaAreaMapUtils
|
||||||
|
import com.fangjia.script.util.MongoUtils
|
||||||
|
import com.mongodb.BasicDBObject
|
||||||
|
import com.mongodb.DBCollection
|
||||||
|
import com.mongodb.client.MongoCollection
|
||||||
|
import net.sf.json.JSONObject
|
||||||
|
import org.apache.commons.collections.CollectionUtils
|
||||||
|
import org.apache.commons.lang.math.RandomUtils
|
||||||
|
import org.apache.commons.lang3.StringUtils
|
||||||
|
import org.bson.Document
|
||||||
|
|
||||||
|
import java.util.regex.Pattern
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 抓取区域边框
|
||||||
|
*/
|
||||||
|
class RegionPoint {
|
||||||
|
|
||||||
|
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
|
||||||
|
static DBCollection GAODECITYCODECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "gaode_city_code")
|
||||||
|
|
||||||
|
static main(args) {
|
||||||
|
def task = new RegionPoint();
|
||||||
|
//河北省、山西省、辽宁省、吉林省、黑龙江省、江苏省、浙江省、
|
||||||
|
// def provinces = ["安徽省","福建省","江西省","山东省","河南省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省"]
|
||||||
|
def provinces = ["内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"]
|
||||||
|
for(def p : provinces){
|
||||||
|
task.crawlCityAndDistrictCode(p);
|
||||||
|
task.run(p);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
def crawlCityAndDistrictCode(def province){
|
||||||
|
crawlCity(province)
|
||||||
|
}
|
||||||
|
|
||||||
|
def run(city){
|
||||||
|
def cityCodes = findCityCode(city);
|
||||||
|
if(cityCodes){
|
||||||
|
for(def cityMap : cityCodes){
|
||||||
|
def codeMap = findAllDistrictCode(cityMap.city_code);
|
||||||
|
def index = 0
|
||||||
|
for(def m : codeMap){
|
||||||
|
index ++
|
||||||
|
def ret = polygon(m.ad_code, "0", "all");
|
||||||
|
if(!ret){
|
||||||
|
println m.name +" " + m.ad_code + "无结果"
|
||||||
|
}
|
||||||
|
println index +" "+cityMap.name+" "+m.name+" "+ret.size()
|
||||||
|
if(ret.size()>0){
|
||||||
|
saveData(cityMap.name, m.name, m.ad_code , ret);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def crawlCity(def province){
|
||||||
|
def ret = polygon(province, "2", "base");
|
||||||
|
if(ret.size()>0){
|
||||||
|
for(def map : ret){
|
||||||
|
if(map.level == "province"){
|
||||||
|
def cityList = map.districts
|
||||||
|
if(cityList && cityList.size()>0){
|
||||||
|
for(def cityMap : cityList){
|
||||||
|
if(cityMap.level == "city"){
|
||||||
|
saveGaoDeCityRegionCode(province, cityMap.name, cityMap.citycode, cityMap.adcode, cityMap.center, cityMap.level);
|
||||||
|
saveGaoDeRegionCode(province, cityMap.districts);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def saveGaoDeRegionCode(province, districts){
|
||||||
|
if(districts){
|
||||||
|
for(def dist : districts){
|
||||||
|
if(dist.level=="district"){
|
||||||
|
saveGaoDeCityRegionCode(province, dist.name, dist.citycode, dist.adcode, dist.center, dist.level);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def findCityCode(city){
|
||||||
|
Pattern pattern = Pattern.compile(city, Pattern.CASE_INSENSITIVE);
|
||||||
|
def q = null;
|
||||||
|
if(city.contains("省") || city.endsWith("自治区")){
|
||||||
|
q = new BasicDBObject("province", pattern);
|
||||||
|
}else{
|
||||||
|
q = new BasicDBObject("name", pattern);
|
||||||
|
}
|
||||||
|
q.put("level", "city")
|
||||||
|
def dbcursor = GAODECITYCODECOLL.find(q)
|
||||||
|
def rets = []
|
||||||
|
for(def d : dbcursor){
|
||||||
|
def city_code = d.get("city_code")
|
||||||
|
if(!rets.contains(city_code)){
|
||||||
|
rets << d
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return rets;
|
||||||
|
}
|
||||||
|
|
||||||
|
private BasicDBObject getLikeStr(String findStr) {
|
||||||
|
Pattern pattern = Pattern.compile(findStr + ".*\$", Pattern.MULTILINE);
|
||||||
|
return new BasicDBObject("$regex", pattern);
|
||||||
|
}
|
||||||
|
|
||||||
|
def findAllDistrictCode(def citycode){
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
q.put("city_code", citycode)
|
||||||
|
q.put("level", "district")
|
||||||
|
def dbcursor = GAODECITYCODECOLL.find(q)
|
||||||
|
def rets = []
|
||||||
|
for(def d : dbcursor){
|
||||||
|
rets << d
|
||||||
|
}
|
||||||
|
return rets;
|
||||||
|
}
|
||||||
|
|
||||||
|
def saveGaoDeCityRegionCode(province, name, citycode, adcode, center, level){
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
q.put("city_code", citycode)
|
||||||
|
q.put("province", province)
|
||||||
|
q.put("level", level)
|
||||||
|
q.put("name", name)
|
||||||
|
if(adcode){
|
||||||
|
q.put("ad_code", adcode)
|
||||||
|
}
|
||||||
|
|
||||||
|
def o = GAODECITYCODECOLL.findOne(q)
|
||||||
|
if(!o){
|
||||||
|
q.put("c_date", new Date())
|
||||||
|
q.put("center", center)
|
||||||
|
GAODECITYCODECOLL.save(q);
|
||||||
|
}else{
|
||||||
|
println name +" "+adcode+" 已存在"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def saveData(city, region, code, ret){
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
q.put("code", code)
|
||||||
|
def dbo = POLYLINECOLL.findOne(q)
|
||||||
|
if(!dbo){
|
||||||
|
def o = new BasicDBObject()
|
||||||
|
o.put("city", city)
|
||||||
|
o.put("region", region)
|
||||||
|
o.put("code", code)
|
||||||
|
o.put("center", ret.center.get(0))
|
||||||
|
o.put("level", ret.level.get(0))
|
||||||
|
o.put("polyline", ret.polyline.get(0));
|
||||||
|
POLYLINECOLL.save(o);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public static def polygon(def keyword, def subdistrict, def extensions) {
|
||||||
|
String url = "https://restapi.amap.com/v3/config/district";
|
||||||
|
|
||||||
|
def params = [:]
|
||||||
|
params.put("keywords", keyword)
|
||||||
|
params.put("subdistrict",subdistrict)
|
||||||
|
params.put("extensions",extensions)
|
||||||
|
|
||||||
|
String html = null;
|
||||||
|
def jo = null;
|
||||||
|
int errorCount = 0;
|
||||||
|
String status = "error"
|
||||||
|
def resultMap = [:]
|
||||||
|
try {
|
||||||
|
HttpRequest http = new HttpRequest();
|
||||||
|
while (errorCount<5){//"2af0b8533415a612b98d59bb93d664b9" 公司key
|
||||||
|
def key = getRandomKey2() //
|
||||||
|
params.put("key", key)
|
||||||
|
html = http.get(url, params,"utf-8")
|
||||||
|
if (StringUtils.isNotEmpty(html)) {
|
||||||
|
jo = JSONObject.fromObject(html)
|
||||||
|
if(jo.infocode!="10000"){
|
||||||
|
errorCount ++
|
||||||
|
Thread.sleep(30*1000)
|
||||||
|
// println " 不可用Key: "+key
|
||||||
|
}else{
|
||||||
|
errorCount = 100
|
||||||
|
status = "normal"
|
||||||
|
// println " 可用Key: "+key + " "+html
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
|
||||||
|
return jo.districts;
|
||||||
|
}
|
||||||
|
|
||||||
|
static String getRandomKey2() {
|
||||||
|
def KeyList = [
|
||||||
|
"2af0b8533415a612b98d59bb93d664b9",
|
||||||
|
"0246c57bbcf24aa246c207b3c99dfe41",
|
||||||
|
"042328b3f0fa5daf426802d379521d87",
|
||||||
|
"055d4f1b5b64a2bce18f4fe1f15c2639",
|
||||||
|
"d5484f5170502405d88bbaffe1b496f0",
|
||||||
|
"e74ccc4258079d2200199695110cceb1",
|
||||||
|
"f58629fc799fbc8537ca85046c6d44a5",
|
||||||
|
"291073f17ee0b963ccb1927ed92bf265",
|
||||||
|
"2a1fb75990f2b39f6221e50892dd3988"
|
||||||
|
]
|
||||||
|
|
||||||
|
def count = RandomUtils.nextInt(KeyList.size())
|
||||||
|
return KeyList.get(count);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
H
|
||||||
Reference in New Issue
Block a user