分批处理
This commit is contained in:
+17
-5
@@ -14,6 +14,7 @@ import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.TimeUnit
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class PoiWash {
|
||||
@@ -256,6 +257,7 @@ class PoiWash {
|
||||
def findPois(def city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("test", 1)
|
||||
// q.put("name", '(三星)企业设立登记第二受理点')
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
@@ -268,12 +270,13 @@ class PoiWash {
|
||||
keys.put("address", 1)
|
||||
keys.put("fjw_city", 1)
|
||||
|
||||
def dbcursor = MAPPOI_CONN.find(q).batchSize(10000)
|
||||
def dbcursor = MAPPOI_CONN.find(q).batchSize(5000)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
if(rets.size()%100000==0){
|
||||
if(rets.size()>=300000){
|
||||
print "."
|
||||
break;
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
@@ -394,10 +397,17 @@ class PoiWash {
|
||||
|
||||
def check(){
|
||||
// "漳州市","三明市","宁德市","泉州市","厦门市","福州市","南平市","龙岩市","莆田市","重庆市","深圳市","北京市","广州市","杭州市","苏州市","成都市",
|
||||
def citys = ["上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市"]
|
||||
def citys = ["重庆市"]
|
||||
for(def city : citys){
|
||||
def datas = findPois(city)
|
||||
processUpdate(city, datas)
|
||||
def stop = false
|
||||
while(!stop){
|
||||
def datas = findPois(city)
|
||||
if(datas.size()>0){
|
||||
processUpdate(city, datas)
|
||||
}else{
|
||||
stop = true
|
||||
}
|
||||
}
|
||||
println city +" 完成"
|
||||
}
|
||||
}
|
||||
@@ -475,6 +485,7 @@ class PoiWash {
|
||||
|
||||
if(udata){
|
||||
updateTotalCount.addAndGet(1)
|
||||
udata.put("test", 2)
|
||||
MAPPOI_CONN.update(new BasicDBObject('id',map.id),new BasicDBObject('$set', udata))
|
||||
println updateTotalCount.get()+" / "+totalCount.get()+" "+count +" "+city
|
||||
}
|
||||
@@ -508,6 +519,7 @@ class PoiWash {
|
||||
}
|
||||
|
||||
latch.await()
|
||||
latch.await(3, TimeUnit.MINUTES);
|
||||
// return rets;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user