优化多线程
This commit is contained in:
+12
-1
@@ -14,6 +14,7 @@ import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
@@ -33,6 +34,7 @@ class GetPoiFromCommunity {
|
||||
AtomicLong atomicLong = new AtomicLong(0)
|
||||
Map<String, String> map = new ConcurrentHashMap<>()
|
||||
def rets = this.findCommunities()
|
||||
CountDownLatch countDownLatch = new CountDownLatch(rets.size())
|
||||
for (def retTemp : rets) {
|
||||
final def ret = retTemp
|
||||
executorService.submit(new Runnable() {
|
||||
@@ -59,6 +61,7 @@ class GetPoiFromCommunity {
|
||||
}
|
||||
synchronized (map) {
|
||||
map.put(id, id)
|
||||
countDownLatch.countDown()
|
||||
if (map.size() % 100 == 0) {
|
||||
atomicLong.addAndGet(map.size())
|
||||
List<String> ids = map.values().stream().collect(Collectors.toList())
|
||||
@@ -70,7 +73,15 @@ class GetPoiFromCommunity {
|
||||
}
|
||||
})
|
||||
}
|
||||
executorService.shutdown()
|
||||
countDownLatch.await()
|
||||
if (map.size() > 0) {
|
||||
atomicLong.addAndGet(map.size())
|
||||
List<String> ids = map.values().stream().collect(Collectors.toList())
|
||||
EsUtils.bulkUpdateFieldByIds("community", "crawl_flag", 0, ids)
|
||||
map.clear()
|
||||
println(atomicLong.intValue())
|
||||
}
|
||||
System.exit(1)
|
||||
}
|
||||
|
||||
static def findCommunities(){
|
||||
|
||||
Reference in New Issue
Block a user