优化
This commit is contained in:
+9
-7
@@ -32,6 +32,7 @@ import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.nio.charset.StandardCharsets
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
@@ -184,8 +185,8 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
}
|
||||
|
||||
def createVillageData(city, pois){
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def keyRets = new CopyOnWriteArrayList()
|
||||
def rets = new ConcurrentHashMap()
|
||||
//def keyRets = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(pois.size())
|
||||
// def townMap = findTownRange(city)
|
||||
@@ -204,9 +205,10 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
def prov = ProvinceData.getProvince(p.city.replace("市",""))
|
||||
|
||||
def ret = createVillageData(prov, p, townMap)
|
||||
if(!keyRets.contains(ret.city+"##"+ret.district+"##"+ret.name)){
|
||||
keyRets << ret.city+"##"+ret.district+"##"+ret.name
|
||||
rets << ret
|
||||
if(!rets.containsKey(ret.city+"##"+ret.district+"##"+ret.name)){
|
||||
//keyRets << ret.city+"##"+ret.district+"##"+ret.name
|
||||
//rets << ret
|
||||
rets.put(ret.city+"##"+ret.district+"##"+ret.name, ret)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -693,8 +695,8 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
int batchSize = 1000;
|
||||
int processedCount = 0;
|
||||
def client = ElasticUtils89.getClient("192.168.0.89")
|
||||
for (int i = 0; i < rows.size(); i++) {
|
||||
def map = rows[i]
|
||||
for (String key : rows.keySet()) {
|
||||
def map = rows.get(key)
|
||||
request.add(new IndexRequest("didata_map_poi_single", "_doc", map.id).source(map, XContentType.JSON));
|
||||
if(i%1000==0){
|
||||
print "."
|
||||
|
||||
Reference in New Issue
Block a user