抓小区

This commit is contained in:
2021-04-22 13:36:39 +08:00
parent 769348b7f4
commit 22765aab5a
@@ -42,10 +42,10 @@ class GaoDeMapCrawl {
def updateErrorLists = []
def flagLists = []
def index = 0
def prefix = "road_"
def prefix = "district_"
// def categorys = ["141202", "141203"]//中学,小学
// def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//住宅相关
def categorys = ["190301"]//道路相关
def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//住宅相关
// def categorys = ["190301"]//道路相关
static main(args) {
def task = new GaoDeMapCrawl();
@@ -109,8 +109,8 @@ class GaoDeMapCrawl {
def run(){//,"西安","重庆"
// def citys = [ "临沂","泰安","菏泽","东营","德州","济宁","日照","聊城","滨州","枣庄","莱芜","济南","青岛","威海","烟台","潍坊","淄博"]
// def citys = ["北京","广州","深圳","合肥","苏州","西安","重庆"]
def citys = CommonUtils.findAllCity()
def citys = ["重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口"]
// def citys = CommonUtils.findAllCity()
def totalCount = 0;
for(def city : citys){
def cityLations = findAllLation(city);
@@ -198,9 +198,11 @@ class GaoDeMapCrawl {
if(m.rets){
processData(m.rets);
location.count = m.rets.size()
location.content = m.toString()
flagLists.add(location)
}else{
location.count = 0
location.content = m.toString()
flagLists.add(location)
}
@@ -336,7 +338,7 @@ class GaoDeMapCrawl {
for(def location : updateLists){
if(location && location._id){
Document updateDocument = new Document();
def doc = new Document(prefix+"status","完成").append("m_date", new Date())
def doc = new Document(prefix+"status","完成").append("m_date", new Date()).append("content", location.content)
updateDocument.append('$set', doc);
Bson updateBson = Filters.eq("_id", location._id);
UpdateOneModel<Document> updateOneModel = new UpdateOneModel<Document>(updateBson, updateDocument);
@@ -358,7 +360,7 @@ class GaoDeMapCrawl {
for(def location : flagLists){
if(location && location._id){
Document updateDocument = new Document();
def doc = new Document(prefix+"gd_data_count",location.count).append("m_date", new Date())
def doc = new Document(prefix+"gd_data_count",location.count).append("m_date", new Date()).append("content", location.content)
updateDocument.append('$set', doc);
Bson updateBson = Filters.eq("_id", location._id);
UpdateOneModel<Document> updateOneModel = new UpdateOneModel<Document>(updateBson, updateDocument);
@@ -382,7 +384,7 @@ class GaoDeMapCrawl {
for(def location : updateErrorLists){
if(location && location._id){
Document updateDocument = new Document();
def doc = new Document(prefix+"status","失败").append("m_date", new Date())
def doc = new Document(prefix+"status","失败").append("m_date", new Date()).append("content", location.content)
updateDocument.append('$set', doc);
Bson updateBson = Filters.eq("_id", location._id);
UpdateOneModel<Document> updateOneModel = new UpdateOneModel<Document>(updateBson, updateDocument);