旧poi合并
This commit is contained in:
+166
-2
@@ -1,5 +1,6 @@
|
|||||||
package com.fangjia.script.gj.data_handle
|
package com.fangjia.script.gj.data_handle
|
||||||
|
|
||||||
|
import com.fangjia.core.util.GeoUtils
|
||||||
import com.fangjia.script.BaseBatch
|
import com.fangjia.script.BaseBatch
|
||||||
import com.fangjia.script.BatchOwner
|
import com.fangjia.script.BatchOwner
|
||||||
import com.fangjia.script.Watcher
|
import com.fangjia.script.Watcher
|
||||||
@@ -8,13 +9,19 @@ import com.fangjia.script.common.DataHandleUtils
|
|||||||
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
||||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||||
import com.fangjia.script.util.ExcelUtils
|
import com.fangjia.script.util.ExcelUtils
|
||||||
|
import com.fangjia.script.util.ProvinceData
|
||||||
|
import com.mongodb.BasicDBObject
|
||||||
|
import org.apache.commons.io.FileUtils
|
||||||
import org.elasticsearch.action.bulk.BulkRequest
|
import org.elasticsearch.action.bulk.BulkRequest
|
||||||
import org.elasticsearch.action.bulk.BulkResponse
|
import org.elasticsearch.action.bulk.BulkResponse
|
||||||
import org.elasticsearch.action.index.IndexRequest
|
import org.elasticsearch.action.index.IndexRequest
|
||||||
import org.elasticsearch.action.support.WriteRequest
|
import org.elasticsearch.action.support.WriteRequest
|
||||||
import org.elasticsearch.client.RequestOptions
|
import org.elasticsearch.client.RequestOptions
|
||||||
import org.elasticsearch.common.xcontent.XContentType
|
import org.elasticsearch.common.xcontent.XContentType
|
||||||
|
import org.elasticsearch.index.query.QueryBuilder
|
||||||
|
import org.elasticsearch.index.query.QueryBuilders
|
||||||
|
|
||||||
|
import java.util.concurrent.ConcurrentHashMap
|
||||||
import java.util.concurrent.CopyOnWriteArrayList
|
import java.util.concurrent.CopyOnWriteArrayList
|
||||||
import java.util.concurrent.CountDownLatch
|
import java.util.concurrent.CountDownLatch
|
||||||
import java.util.concurrent.ExecutorService
|
import java.util.concurrent.ExecutorService
|
||||||
@@ -39,8 +46,8 @@ class NewGaoDePoiBatch extends BaseBatch{
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
void process(Object args) {
|
void process(Object args) {
|
||||||
run()
|
// run()
|
||||||
// oldPoiMagre()
|
oldPoiMagre()
|
||||||
}
|
}
|
||||||
|
|
||||||
def run(){
|
def run(){
|
||||||
@@ -349,7 +356,164 @@ class NewGaoDePoiBatch extends BaseBatch{
|
|||||||
}
|
}
|
||||||
|
|
||||||
def oldPoiMagre(){
|
def oldPoiMagre(){
|
||||||
|
def cityList = CommonDatautils.findFullNameCitys().reverse()
|
||||||
|
for(def cityObj : cityList){
|
||||||
|
// if(!cityObj.city.contains("固原")){
|
||||||
|
// continue
|
||||||
|
// }
|
||||||
|
def fileName = "/home/goojia/gj/poi/poi/${cityObj.city}.csv"
|
||||||
|
if (CommonDatautils.isWindow()) {
|
||||||
|
fileName = "D:\\数据\\merge_poi\\${cityObj.city}.csv"
|
||||||
|
}
|
||||||
|
File file=new File(fileName);
|
||||||
|
if(file.exists()){
|
||||||
|
def results = readCSV(fileName)
|
||||||
|
println cityObj.province+" " +cityObj.city+" "+results.size()
|
||||||
|
if(results){
|
||||||
|
def megerMap = poiRepeat(cityObj.city, results)
|
||||||
|
save(megerMap.values())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def poiRepeat(city, oldPois){
|
||||||
|
def rets = new ConcurrentHashMap()
|
||||||
|
|
||||||
|
def totalCount = new AtomicLong()
|
||||||
|
final CountDownLatch latch = new CountDownLatch(oldPois.size())
|
||||||
|
def townMap = CommonDatautils.queryTownByCity(city)
|
||||||
|
def esMap = findEsPoiByCity(city)
|
||||||
|
|
||||||
|
println "Es 加载完成:"+new Date().format("yyyy-MM-dd HH:mm:ss")
|
||||||
|
for(def i=0;i<oldPois.size();i++){
|
||||||
|
final def p = oldPois[i]
|
||||||
|
fixedThreadPool.execute(new Runnable() {
|
||||||
|
void run() {
|
||||||
|
try{
|
||||||
|
totalCount.addAndGet(1)
|
||||||
|
def nameMap = esMap.get(p.district)
|
||||||
|
def nameKeys = nameMap.keySet()
|
||||||
|
def key = p.name.replace("-","").replace("·","")
|
||||||
|
if(!nameKeys.contains(key)){
|
||||||
|
def simlarObj = null
|
||||||
|
if(esMap){
|
||||||
|
simlarObj = findSimlarPoi(p, esMap)
|
||||||
|
}
|
||||||
|
if(!simlarObj){
|
||||||
|
def newKey = p.city+"##"+p.district+"##"+p.name.replace("-","").replace("·","")
|
||||||
|
if(!rets.containsKey(newKey)){
|
||||||
|
p.data_source_year = "2023-08"
|
||||||
|
rets.put(newKey, p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
e.printStackTrace()
|
||||||
|
} finally {
|
||||||
|
latch.countDown()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
latch.await()
|
||||||
|
return rets
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def findSimlarPoi(ret, esMap){
|
||||||
|
def simlarObj = null
|
||||||
|
def region = ""
|
||||||
|
if(ret.district){
|
||||||
|
region = ret.district
|
||||||
|
}
|
||||||
|
def regionMap = esMap.get(region)
|
||||||
|
if(regionMap){
|
||||||
|
def keys = regionMap.keySet()
|
||||||
|
|
||||||
|
for(def key : keys){
|
||||||
|
def esValue = regionMap.get(key)
|
||||||
|
if(esValue.township && ret.township){
|
||||||
|
if(esValue.name.contains(ret.name) || ret.name.contains(esValue.name)){//||
|
||||||
|
def distance = GeoUtils.calculateDistance(ret.location.lon, ret.location.lat, esValue.location.lon, esValue.location.lat);
|
||||||
|
if(distance <100){
|
||||||
|
simlarObj = esValue
|
||||||
|
println esValue.name +" "+ret.name+" "+distance
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}else{
|
||||||
|
println "无数据"
|
||||||
|
}
|
||||||
|
|
||||||
|
return simlarObj
|
||||||
|
}
|
||||||
|
|
||||||
|
def findEsPoiByCity(city){
|
||||||
|
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
|
||||||
|
def map = new HashMap()
|
||||||
|
ElasticUtils89.executeQuery("192.168.0.89", "didata_map_poi", queryBuilder, { String id, Map d->
|
||||||
|
def district = ""
|
||||||
|
if(d.district){
|
||||||
|
district = d.district
|
||||||
|
}
|
||||||
|
def regionMap = map.get(district)
|
||||||
|
if(!regionMap){
|
||||||
|
regionMap = new HashMap()
|
||||||
|
}
|
||||||
|
def o = new BasicDBObject()
|
||||||
|
o.put("city", d.city)
|
||||||
|
o.put("district", d.district)
|
||||||
|
o.put("name", d.name)
|
||||||
|
o.put("township", d.township)
|
||||||
|
o.put("location", d.location)
|
||||||
|
regionMap.put(d.name.replace("-","").replace("·",""), o)
|
||||||
|
map.put(district, regionMap)
|
||||||
|
})
|
||||||
|
return map
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def readCSV(filename){
|
||||||
|
def lines = FileUtils.readLines(new File(filename))
|
||||||
|
def rets = [];
|
||||||
|
for(line in lines) {
|
||||||
|
if(line){
|
||||||
|
String[] arrs = line.split("\\|@\\|")
|
||||||
|
def o = new BasicDBObject()
|
||||||
|
o.put("province",arrs[0])
|
||||||
|
o.put("city" , arrs[1])
|
||||||
|
o.put("district",arrs[2])
|
||||||
|
o.put("name",arrs[3])
|
||||||
|
o.put("township",arrs[4])
|
||||||
|
o.put("address",arrs[5])
|
||||||
|
o.put("cat",arrs[6])
|
||||||
|
o.put("type",arrs[7])
|
||||||
|
o.put("secondary_type",arrs[8])
|
||||||
|
o.put("small_type" ,arrs[9])
|
||||||
|
if(arrs[10]){
|
||||||
|
o.put("data_id",arrs[10])
|
||||||
|
}else{
|
||||||
|
o.put("data_id",arrs[11])
|
||||||
|
}
|
||||||
|
|
||||||
|
o.put("spell_name",arrs[12])
|
||||||
|
def location = [:]
|
||||||
|
if(arrs[13]){
|
||||||
|
location.lon = arrs[13].toString().toDouble()
|
||||||
|
location.lat = arrs[14].toString().toDouble()
|
||||||
|
}
|
||||||
|
o.put("location",location)
|
||||||
|
o.put("data_source",arrs[15])
|
||||||
|
o.put("status",arrs[16])
|
||||||
|
o.put("version",arrs[17])
|
||||||
|
rets << o
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return rets;
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user