分批处理

This commit is contained in:
2023-10-17 14:56:36 +08:00
parent 5782999d08
commit 2ac10a06f3
@@ -14,6 +14,7 @@ import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicLong
class PoiWash {
@@ -256,6 +257,7 @@ class PoiWash {
def findPois(def city){
def q = new BasicDBObject()
q.put("city", city)
q.put("test", 1)
// q.put("name", '(三星)企业设立登记第二受理点')
def keys = new BasicDBObject()
@@ -268,12 +270,13 @@ class PoiWash {
keys.put("address", 1)
keys.put("fjw_city", 1)
def dbcursor = MAPPOI_CONN.find(q).batchSize(10000)
def dbcursor = MAPPOI_CONN.find(q).batchSize(5000)
def rets = []
for(def d : dbcursor){
rets << d
if(rets.size()%100000==0){
if(rets.size()>=300000){
print "."
break;
}
}
return rets;
@@ -394,10 +397,17 @@ class PoiWash {
def check(){
// "漳州市","三明市","宁德市","泉州市","厦门市","福州市","南平市","龙岩市","莆田市","重庆市","深圳市","北京市","广州市","杭州市","苏州市","成都市",
def citys = ["上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市"]
def citys = ["重庆市"]
for(def city : citys){
def datas = findPois(city)
processUpdate(city, datas)
def stop = false
while(!stop){
def datas = findPois(city)
if(datas.size()>0){
processUpdate(city, datas)
}else{
stop = true
}
}
println city +" 完成"
}
}
@@ -475,6 +485,7 @@ class PoiWash {
if(udata){
updateTotalCount.addAndGet(1)
udata.put("test", 2)
MAPPOI_CONN.update(new BasicDBObject('id',map.id),new BasicDBObject('$set', udata))
println updateTotalCount.get()+" / "+totalCount.get()+" "+count +" "+city
}
@@ -508,6 +519,7 @@ class PoiWash {
}
latch.await()
latch.await(3, TimeUnit.MINUTES);
// return rets;
}