地图抓取修改

This commit is contained in:
2021-06-10 14:24:54 +08:00
parent c5800012df
commit 809bfcb16f
@@ -127,7 +127,7 @@ class GaoDeMapCrawl extends BaseBatch{
q.put(prefix+"status", new BasicDBObject(QueryOperators.EXISTS, false))
// q.put("status", "失败")
// q.put("traffic_status", new BasicDBObject(QueryOperators.EXISTS, false))
def results = BOUNDS_COLL.find(q).batchSize(2000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def results = BOUNDS_COLL.find(q).limit(2000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
return results;
}
@@ -140,7 +140,7 @@ class GaoDeMapCrawl extends BaseBatch{
// void run() {
// try{
// totalCount.addAndGet(1)
searching(location)
searching(i, location)
// }catch(Exception e){
// e.printStackTrace()
// } finally {
@@ -167,7 +167,7 @@ class GaoDeMapCrawl extends BaseBatch{
}
}
def searching(location){
def searching(i, location){
// def bound_gd = location.bound_gd
// def lats = [bound_gd[1], bound_gd[3]]
// def lngs = [bound_gd[0], bound_gd[2]]
@@ -175,7 +175,7 @@ class GaoDeMapCrawl extends BaseBatch{
// lngs.sort()
// println index
// if(lngs[0]<=121.418153 && lngs[1]>=121.418153 && lats[0]<=31.218824 && lats[1]>=31.218824){
search(location)
search(i, location)
// }
if(updateLists.size() > 0 && updateLists.size()>1000){
@@ -194,7 +194,7 @@ class GaoDeMapCrawl extends BaseBatch{
}
def search(location){
def search(i, location){
def bound_gd = location.bound_gd
def lats = [bound_gd[1], bound_gd[3]]
def lngs = [bound_gd[0], bound_gd[2]]
@@ -228,7 +228,7 @@ class GaoDeMapCrawl extends BaseBatch{
updateErrorLists.addAll(location)
}
// update(location);
println "[ "+totalCount.get()+" ]"+location._id+" "+location.city+" "+location.bound_gd+" flag: "+m.status
println "[ "+i+" ]"+location._id+" "+location.city+" "+location.bound_gd+" flag: "+m.status
}
def processData(rets){