This commit is contained in:
沈健
2024-08-23 10:06:04 +08:00
parent b22208aaa1
commit 1dde692401
@@ -32,6 +32,7 @@ import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.nio.charset.StandardCharsets
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
@@ -184,8 +185,8 @@ class GaoDePoiDataProcess extends BaseBatch{
}
def createVillageData(city, pois){
def rets = new CopyOnWriteArrayList()
def keyRets = new CopyOnWriteArrayList()
def rets = new ConcurrentHashMap()
//def keyRets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(pois.size())
// def townMap = findTownRange(city)
@@ -204,9 +205,10 @@ class GaoDePoiDataProcess extends BaseBatch{
def prov = ProvinceData.getProvince(p.city.replace("市",""))
def ret = createVillageData(prov, p, townMap)
if(!keyRets.contains(ret.city+"##"+ret.district+"##"+ret.name)){
keyRets << ret.city+"##"+ret.district+"##"+ret.name
rets << ret
if(!rets.containsKey(ret.city+"##"+ret.district+"##"+ret.name)){
//keyRets << ret.city+"##"+ret.district+"##"+ret.name
//rets << ret
rets.put(ret.city+"##"+ret.district+"##"+ret.name, ret)
}
}
@@ -693,8 +695,8 @@ class GaoDePoiDataProcess extends BaseBatch{
int batchSize = 1000;
int processedCount = 0;
def client = ElasticUtils89.getClient("192.168.0.89")
for (int i = 0; i < rows.size(); i++) {
def map = rows[i]
for (String key : rows.keySet()) {
def map = rows.get(key)
request.add(new IndexRequest("didata_map_poi_single", "_doc", map.id).source(map, XContentType.JSON));
if(i%1000==0){
print "."