旧poi合并

This commit is contained in:
2024-12-19 11:19:30 +08:00
parent 110fc4003f
commit 02eeab9399
@@ -1,5 +1,6 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
@@ -8,13 +9,19 @@ import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.ProvinceData
import com.mongodb.BasicDBObject
import org.apache.commons.io.FileUtils
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.action.support.WriteRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.common.xcontent.XContentType
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
@@ -39,8 +46,8 @@ class NewGaoDePoiBatch extends BaseBatch{
@Override
void process(Object args) {
run()
// oldPoiMagre()
// run()
oldPoiMagre()
}
def run(){
@@ -349,7 +356,164 @@ class NewGaoDePoiBatch extends BaseBatch{
}
def oldPoiMagre(){
def cityList = CommonDatautils.findFullNameCitys().reverse()
for(def cityObj : cityList){
// if(!cityObj.city.contains("固原")){
// continue
// }
def fileName = "/home/goojia/gj/poi/poi/${cityObj.city}.csv"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\merge_poi\\${cityObj.city}.csv"
}
File file=new File(fileName);
if(file.exists()){
def results = readCSV(fileName)
println cityObj.province+" " +cityObj.city+" "+results.size()
if(results){
def megerMap = poiRepeat(cityObj.city, results)
save(megerMap.values())
}
}
}
}
def poiRepeat(city, oldPois){
def rets = new ConcurrentHashMap()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(oldPois.size())
def townMap = CommonDatautils.queryTownByCity(city)
def esMap = findEsPoiByCity(city)
println "Es 加载完成:"+new Date().format("yyyy-MM-dd HH:mm:ss")
for(def i=0;i<oldPois.size();i++){
final def p = oldPois[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def nameMap = esMap.get(p.district)
def nameKeys = nameMap.keySet()
def key = p.name.replace("-","").replace("·","")
if(!nameKeys.contains(key)){
def simlarObj = null
if(esMap){
simlarObj = findSimlarPoi(p, esMap)
}
if(!simlarObj){
def newKey = p.city+"##"+p.district+"##"+p.name.replace("-","").replace("·","")
if(!rets.containsKey(newKey)){
p.data_source_year = "2023-08"
rets.put(newKey, p)
}
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets
}
def findSimlarPoi(ret, esMap){
def simlarObj = null
def region = ""
if(ret.district){
region = ret.district
}
def regionMap = esMap.get(region)
if(regionMap){
def keys = regionMap.keySet()
for(def key : keys){
def esValue = regionMap.get(key)
if(esValue.township && ret.township){
if(esValue.name.contains(ret.name) || ret.name.contains(esValue.name)){//||
def distance = GeoUtils.calculateDistance(ret.location.lon, ret.location.lat, esValue.location.lon, esValue.location.lat);
if(distance <100){
simlarObj = esValue
println esValue.name +" "+ret.name+" "+distance
break;
}
}
}
}
}else{
println "无数据"
}
return simlarObj
}
def findEsPoiByCity(city){
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
def map = new HashMap()
ElasticUtils89.executeQuery("192.168.0.89", "didata_map_poi", queryBuilder, { String id, Map d->
def district = ""
if(d.district){
district = d.district
}
def regionMap = map.get(district)
if(!regionMap){
regionMap = new HashMap()
}
def o = new BasicDBObject()
o.put("city", d.city)
o.put("district", d.district)
o.put("name", d.name)
o.put("township", d.township)
o.put("location", d.location)
regionMap.put(d.name.replace("-","").replace("·",""), o)
map.put(district, regionMap)
})
return map
}
def readCSV(filename){
def lines = FileUtils.readLines(new File(filename))
def rets = [];
for(line in lines) {
if(line){
String[] arrs = line.split("\\|@\\|")
def o = new BasicDBObject()
o.put("province",arrs[0])
o.put("city" , arrs[1])
o.put("district",arrs[2])
o.put("name",arrs[3])
o.put("township",arrs[4])
o.put("address",arrs[5])
o.put("cat",arrs[6])
o.put("type",arrs[7])
o.put("secondary_type",arrs[8])
o.put("small_type" ,arrs[9])
if(arrs[10]){
o.put("data_id",arrs[10])
}else{
o.put("data_id",arrs[11])
}
o.put("spell_name",arrs[12])
def location = [:]
if(arrs[13]){
location.lon = arrs[13].toString().toDouble()
location.lat = arrs[14].toString().toDouble()
}
o.put("location",location)
o.put("data_source",arrs[15])
o.put("status",arrs[16])
o.put("version",arrs[17])
rets << o
}
}
return rets;
}
}