diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/PoiWash.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/PoiWash.groovy index 8a7f8daf1..f9b09dac1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/PoiWash.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/PoiWash.groovy @@ -14,6 +14,7 @@ import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicLong class PoiWash { @@ -256,6 +257,7 @@ class PoiWash { def findPois(def city){ def q = new BasicDBObject() q.put("city", city) + q.put("test", 1) // q.put("name", '(三星)企业设立登记第二受理点') def keys = new BasicDBObject() @@ -268,12 +270,13 @@ class PoiWash { keys.put("address", 1) keys.put("fjw_city", 1) - def dbcursor = MAPPOI_CONN.find(q).batchSize(10000) + def dbcursor = MAPPOI_CONN.find(q).batchSize(5000) def rets = [] for(def d : dbcursor){ rets << d - if(rets.size()%100000==0){ + if(rets.size()>=300000){ print "." + break; } } return rets; @@ -394,10 +397,17 @@ class PoiWash { def check(){ // "漳州市","三明市","宁德市","泉州市","厦门市","福州市","南平市","龙岩市","莆田市","重庆市","深圳市","北京市","广州市","杭州市","苏州市","成都市", - def citys = ["上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市"] + def citys = ["重庆市"] for(def city : citys){ - def datas = findPois(city) - processUpdate(city, datas) + def stop = false + while(!stop){ + def datas = findPois(city) + if(datas.size()>0){ + processUpdate(city, datas) + }else{ + stop = true + } + } println city +" 完成" } } @@ -475,6 +485,7 @@ class PoiWash { if(udata){ updateTotalCount.addAndGet(1) + udata.put("test", 2) MAPPOI_CONN.update(new BasicDBObject('id',map.id),new BasicDBObject('$set', udata)) println updateTotalCount.get()+" / "+totalCount.get()+" "+count +" "+city } @@ -508,6 +519,7 @@ class PoiWash { } latch.await() + latch.await(3, TimeUnit.MINUTES); // return rets; }