+166
-2
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
@@ -8,13 +9,19 @@ import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.ProvinceData
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.apache.commons.io.FileUtils
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
import org.elasticsearch.action.index.IndexRequest
|
||||
import org.elasticsearch.action.support.WriteRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.common.xcontent.XContentType
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
@@ -39,8 +46,8 @@ class NewGaoDePoiBatch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
run()
|
||||
// oldPoiMagre()
|
||||
// run()
|
||||
oldPoiMagre()
|
||||
}
|
||||
|
||||
def run(){
|
||||
@@ -349,7 +356,164 @@ class NewGaoDePoiBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
def oldPoiMagre(){
|
||||
def cityList = CommonDatautils.findFullNameCitys().reverse()
|
||||
for(def cityObj : cityList){
|
||||
// if(!cityObj.city.contains("固原")){
|
||||
// continue
|
||||
// }
|
||||
def fileName = "/home/goojia/gj/poi/poi/${cityObj.city}.csv"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\merge_poi\\${cityObj.city}.csv"
|
||||
}
|
||||
File file=new File(fileName);
|
||||
if(file.exists()){
|
||||
def results = readCSV(fileName)
|
||||
println cityObj.province+" " +cityObj.city+" "+results.size()
|
||||
if(results){
|
||||
def megerMap = poiRepeat(cityObj.city, results)
|
||||
save(megerMap.values())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def poiRepeat(city, oldPois){
|
||||
def rets = new ConcurrentHashMap()
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(oldPois.size())
|
||||
def townMap = CommonDatautils.queryTownByCity(city)
|
||||
def esMap = findEsPoiByCity(city)
|
||||
|
||||
println "Es 加载完成:"+new Date().format("yyyy-MM-dd HH:mm:ss")
|
||||
for(def i=0;i<oldPois.size();i++){
|
||||
final def p = oldPois[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def nameMap = esMap.get(p.district)
|
||||
def nameKeys = nameMap.keySet()
|
||||
def key = p.name.replace("-","").replace("·","")
|
||||
if(!nameKeys.contains(key)){
|
||||
def simlarObj = null
|
||||
if(esMap){
|
||||
simlarObj = findSimlarPoi(p, esMap)
|
||||
}
|
||||
if(!simlarObj){
|
||||
def newKey = p.city+"##"+p.district+"##"+p.name.replace("-","").replace("·","")
|
||||
if(!rets.containsKey(newKey)){
|
||||
p.data_source_year = "2023-08"
|
||||
rets.put(newKey, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets
|
||||
}
|
||||
|
||||
|
||||
def findSimlarPoi(ret, esMap){
|
||||
def simlarObj = null
|
||||
def region = ""
|
||||
if(ret.district){
|
||||
region = ret.district
|
||||
}
|
||||
def regionMap = esMap.get(region)
|
||||
if(regionMap){
|
||||
def keys = regionMap.keySet()
|
||||
|
||||
for(def key : keys){
|
||||
def esValue = regionMap.get(key)
|
||||
if(esValue.township && ret.township){
|
||||
if(esValue.name.contains(ret.name) || ret.name.contains(esValue.name)){//||
|
||||
def distance = GeoUtils.calculateDistance(ret.location.lon, ret.location.lat, esValue.location.lon, esValue.location.lat);
|
||||
if(distance <100){
|
||||
simlarObj = esValue
|
||||
println esValue.name +" "+ret.name+" "+distance
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}else{
|
||||
println "无数据"
|
||||
}
|
||||
|
||||
return simlarObj
|
||||
}
|
||||
|
||||
def findEsPoiByCity(city){
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
|
||||
def map = new HashMap()
|
||||
ElasticUtils89.executeQuery("192.168.0.89", "didata_map_poi", queryBuilder, { String id, Map d->
|
||||
def district = ""
|
||||
if(d.district){
|
||||
district = d.district
|
||||
}
|
||||
def regionMap = map.get(district)
|
||||
if(!regionMap){
|
||||
regionMap = new HashMap()
|
||||
}
|
||||
def o = new BasicDBObject()
|
||||
o.put("city", d.city)
|
||||
o.put("district", d.district)
|
||||
o.put("name", d.name)
|
||||
o.put("township", d.township)
|
||||
o.put("location", d.location)
|
||||
regionMap.put(d.name.replace("-","").replace("·",""), o)
|
||||
map.put(district, regionMap)
|
||||
})
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
def readCSV(filename){
|
||||
def lines = FileUtils.readLines(new File(filename))
|
||||
def rets = [];
|
||||
for(line in lines) {
|
||||
if(line){
|
||||
String[] arrs = line.split("\\|@\\|")
|
||||
def o = new BasicDBObject()
|
||||
o.put("province",arrs[0])
|
||||
o.put("city" , arrs[1])
|
||||
o.put("district",arrs[2])
|
||||
o.put("name",arrs[3])
|
||||
o.put("township",arrs[4])
|
||||
o.put("address",arrs[5])
|
||||
o.put("cat",arrs[6])
|
||||
o.put("type",arrs[7])
|
||||
o.put("secondary_type",arrs[8])
|
||||
o.put("small_type" ,arrs[9])
|
||||
if(arrs[10]){
|
||||
o.put("data_id",arrs[10])
|
||||
}else{
|
||||
o.put("data_id",arrs[11])
|
||||
}
|
||||
|
||||
o.put("spell_name",arrs[12])
|
||||
def location = [:]
|
||||
if(arrs[13]){
|
||||
location.lon = arrs[13].toString().toDouble()
|
||||
location.lat = arrs[14].toString().toDouble()
|
||||
}
|
||||
o.put("location",location)
|
||||
o.put("data_source",arrs[15])
|
||||
o.put("status",arrs[16])
|
||||
o.put("version",arrs[17])
|
||||
rets << o
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user