新版poi处理
This commit is contained in:
+31
-3
@@ -4209,13 +4209,13 @@ class CommonDatautils {
|
|||||||
|
|
||||||
|
|
||||||
static def findAllNewPoiDatas(city){
|
static def findAllNewPoiDatas(city){
|
||||||
String sql = "select cityname,adname,name,address,location from poi_amap_2024 where cityname='${city}'"
|
String sql = "select pname,cityname,adname,name,address,location,type,id from poi_amap_2024 where cityname='${city}' limit 100"
|
||||||
def rets = [:]
|
def rets = []
|
||||||
def index= 0
|
def index= 0
|
||||||
ElasticUtils.executeSqlQuery(sql, { map->
|
ElasticUtils.executeSqlQuery(sql, { map->
|
||||||
index++
|
index++
|
||||||
println index
|
println index
|
||||||
rets << ["map_id":map.id,"city":map.cityname, "district":map.adname, "name":map.name, "address":map.address, "type":map.type,"lng":map.location.x, "lat":map.location.y]
|
rets << ["map_id":map.id,"province":map.pname,"city":map.cityname, "district":map.adname, "name":map.name, "address":map.address, "cat":map.type,"lng":map.location.x, "lat":map.location.y]
|
||||||
})
|
})
|
||||||
return rets;
|
return rets;
|
||||||
}
|
}
|
||||||
@@ -5149,6 +5149,34 @@ class CommonDatautils {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
static def queryFullRegionTownByCity(city, fields){
|
||||||
|
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
q.put("type","乡镇街道")
|
||||||
|
q.put("city",city)
|
||||||
|
|
||||||
|
def keys = new BasicDBObject()
|
||||||
|
for(def key : fields){
|
||||||
|
keys.put(key,1)
|
||||||
|
}
|
||||||
|
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||||
|
def townMap = [:]
|
||||||
|
for(def d : dbcursor){
|
||||||
|
if(d.district){
|
||||||
|
def list = townMap.get(d.district)
|
||||||
|
if(!list){
|
||||||
|
list = []
|
||||||
|
}
|
||||||
|
list << d
|
||||||
|
townMap.put(d.district, list)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
println "[${city}] 乡镇街道 加载完成 "+new Date().format("yyyy-MM-dd HH:mm:ss")
|
||||||
|
return townMap
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
static def queryTownByCity(city){
|
static def queryTownByCity(city){
|
||||||
|
|
||||||
|
|||||||
+232
@@ -0,0 +1,232 @@
|
|||||||
|
package com.fangjia.script.gj.data_handle
|
||||||
|
|
||||||
|
import com.fangjia.script.BaseBatch
|
||||||
|
import com.fangjia.script.BatchOwner
|
||||||
|
import com.fangjia.script.Watcher
|
||||||
|
import com.fangjia.script.commData.CommonDatautils
|
||||||
|
import com.fangjia.script.common.DataHandleUtils
|
||||||
|
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
||||||
|
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||||
|
import com.fangjia.script.util.ExcelUtils
|
||||||
|
import org.elasticsearch.action.bulk.BulkRequest
|
||||||
|
import org.elasticsearch.action.bulk.BulkResponse
|
||||||
|
import org.elasticsearch.action.index.IndexRequest
|
||||||
|
import org.elasticsearch.action.support.WriteRequest
|
||||||
|
import org.elasticsearch.client.RequestOptions
|
||||||
|
import org.elasticsearch.common.xcontent.XContentType
|
||||||
|
|
||||||
|
import java.util.concurrent.CopyOnWriteArrayList
|
||||||
|
import java.util.concurrent.CountDownLatch
|
||||||
|
import java.util.concurrent.ExecutorService
|
||||||
|
import java.util.concurrent.Executors
|
||||||
|
import java.util.concurrent.atomic.AtomicLong
|
||||||
|
/**
|
||||||
|
* 新版poi处理入库(第二批)
|
||||||
|
*/
|
||||||
|
@BatchOwner(Watcher.GaoJuan)
|
||||||
|
class NewGaoDePoiBatch extends BaseBatch{
|
||||||
|
|
||||||
|
public static def version = new Date().getTime()
|
||||||
|
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||||
|
|
||||||
|
public static def TAGS = []
|
||||||
|
|
||||||
|
static main(args) {
|
||||||
|
runBatch(new NewGaoDePoiBatch(), args)
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
void process(Object args) {
|
||||||
|
run()
|
||||||
|
}
|
||||||
|
|
||||||
|
def run(){
|
||||||
|
if (CommonDatautils.isWindow()) {
|
||||||
|
TAGS = getAllPoiType("D:\\数据\\福建省_POI_GCJ02_20240630\\POI分类白名单.xlsx")
|
||||||
|
}else{
|
||||||
|
TAGS = getAllPoiType("/home/goojia/gj/POI_TYPES.xlsx")
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def cityList = CommonDatautils.findFullNameCitys().reverse()
|
||||||
|
for(def cityObj : cityList){
|
||||||
|
if(cityObj.city == "上海市"){
|
||||||
|
def results = CommonDatautils.findAllNewPoiDatas(cityObj.city)
|
||||||
|
println cityObj.city+" "+results.size()
|
||||||
|
def rows = running(cityObj.city, results)
|
||||||
|
save(rows)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
def getAllPoiType(filePath){
|
||||||
|
def poiTypes = ExcelUtils.read(filePath, 0, 1)
|
||||||
|
def types = []
|
||||||
|
for(def poi : poiTypes){
|
||||||
|
types << poi['A']
|
||||||
|
}
|
||||||
|
return types
|
||||||
|
}
|
||||||
|
|
||||||
|
def running(city,results){
|
||||||
|
def rets = new CopyOnWriteArrayList()
|
||||||
|
def totalCount = new AtomicLong()
|
||||||
|
final CountDownLatch latch = new CountDownLatch(results.size())
|
||||||
|
def townMap = CommonDatautils.queryFullRegionTownByCity(city, ["city","district","street", "shape"])
|
||||||
|
for(def i=0;i<results.size();i++){
|
||||||
|
def map = results[i]
|
||||||
|
// fixedThreadPool.execute(new Runnable() {
|
||||||
|
// void run() {
|
||||||
|
try{
|
||||||
|
totalCount.addAndGet(1)
|
||||||
|
rets.addAll(createData(map, townMap))
|
||||||
|
} catch (Exception e) {
|
||||||
|
e.printStackTrace()
|
||||||
|
} finally {
|
||||||
|
latch.countDown()
|
||||||
|
}
|
||||||
|
// }
|
||||||
|
// });
|
||||||
|
}
|
||||||
|
// latch.await()
|
||||||
|
return rets;
|
||||||
|
}
|
||||||
|
|
||||||
|
def createData(map, townMap){
|
||||||
|
def saveObj = [:]
|
||||||
|
saveObj.province = map.province
|
||||||
|
saveObj.city = map.city
|
||||||
|
saveObj.district = map.district
|
||||||
|
saveObj.name = map.name
|
||||||
|
if(map.cat){
|
||||||
|
def arrs = map.cat.split(";|:")
|
||||||
|
if(arrs.size()>=1){
|
||||||
|
saveObj.type = arrs[0]
|
||||||
|
}
|
||||||
|
if(arrs.size()>=2){
|
||||||
|
saveObj.secondary_type = arrs[1]
|
||||||
|
}else{
|
||||||
|
saveObj.secondary_type = saveObj.type
|
||||||
|
}
|
||||||
|
if(arrs.size()>=3){
|
||||||
|
def smallstr = arrs[2]
|
||||||
|
def smallArr = null
|
||||||
|
if(smallstr.contains("|")){
|
||||||
|
smallArr = smallstr.split("|")
|
||||||
|
}
|
||||||
|
if(smallArr && smallArr.size()>=1){
|
||||||
|
saveObj.small_type = smallArr[0]
|
||||||
|
}else{
|
||||||
|
saveObj.small_type = smallstr
|
||||||
|
}
|
||||||
|
}else{
|
||||||
|
saveObj.small_type = saveObj.secondary_type
|
||||||
|
}
|
||||||
|
|
||||||
|
saveObj.cat = saveObj.type+";"+saveObj.secondary_type+";"+saveObj.small_type
|
||||||
|
|
||||||
|
}
|
||||||
|
def location = [:]
|
||||||
|
location.lon = DataHandleUtils.getDoubleRound(6, map.lng)
|
||||||
|
location.lat = DataHandleUtils.getDoubleRound(6, map.lat)
|
||||||
|
saveObj.location = location
|
||||||
|
saveObj.spell_name = com.fangjia.core.util.DataUtil.hanyuQuanpin(map.name, " ")
|
||||||
|
saveObj.version = version
|
||||||
|
saveObj.data_id = map.map_id
|
||||||
|
saveObj.data_source = "高德地图"
|
||||||
|
saveObj.data_source_year = "2024-11"
|
||||||
|
|
||||||
|
if(saveObj.cat && TAGS.contains(saveObj.cat)){
|
||||||
|
saveObj.status = 0
|
||||||
|
}else{
|
||||||
|
saveObj.status = 1
|
||||||
|
}
|
||||||
|
|
||||||
|
if(location){
|
||||||
|
def towns = townMap.get(saveObj.district)
|
||||||
|
if(towns){
|
||||||
|
for(def town : towns){
|
||||||
|
if(town.shape && town.shape.coordinates){
|
||||||
|
def flag = InsidePolygon.isInArea(location.lon, location.lat, InsidePolygon.createPoint4(town.shape.coordinates))
|
||||||
|
if(flag){
|
||||||
|
saveObj.township = town.street
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if(!saveObj.township && map.address.contains("街道")){
|
||||||
|
println map.address +" 含街道"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if(map.address){
|
||||||
|
if(map.address.length()<3){
|
||||||
|
saveObj.township = ""
|
||||||
|
}else{
|
||||||
|
def address = map.address
|
||||||
|
def i = 0
|
||||||
|
while(i<5){
|
||||||
|
if(address.toString().startsWith(map.province)){
|
||||||
|
address = address.toString().substring(0, map.province.toString().length())
|
||||||
|
}
|
||||||
|
if(address.toString().startsWith(map.city)){
|
||||||
|
address = address.toString().substring(0, map.city.toString().length())
|
||||||
|
}
|
||||||
|
if(address.toString().startsWith(map.district)){
|
||||||
|
address = address.toString().substring(0, map.district.toString().length())
|
||||||
|
}
|
||||||
|
if(address.toString().startsWith("市辖区")){
|
||||||
|
address = address.toString().substring(0, 3)
|
||||||
|
}
|
||||||
|
if(address.toString().startsWith(saveObj.township)){
|
||||||
|
address = address.toString().substring(0, saveObj.township.toString().length())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
saveObj.township = address
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return saveObj
|
||||||
|
}
|
||||||
|
|
||||||
|
def save(rows){
|
||||||
|
BulkRequest request = new BulkRequest();
|
||||||
|
int batchSize = 1000;
|
||||||
|
int processedCount = 0;
|
||||||
|
def client = ElasticUtils89.getClient("192.168.0.89")
|
||||||
|
|
||||||
|
rows.each{key,map->
|
||||||
|
// def map = rows.get(key)
|
||||||
|
|
||||||
|
request.add(new IndexRequest("didata_map_poi_single", "_doc", map.id).source(map, XContentType.JSON));
|
||||||
|
|
||||||
|
// 批量提交数据
|
||||||
|
if (request.numberOfActions() >= batchSize) {
|
||||||
|
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||||
|
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||||
|
if (bulkResponse.hasFailures()) {
|
||||||
|
// 处理错误
|
||||||
|
print "Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||||
|
} else {
|
||||||
|
processedCount += request.numberOfActions();
|
||||||
|
println "Processed " + processedCount + " documents "+new Date().format("yyyy-MM-dd HH:mm:ss")
|
||||||
|
}
|
||||||
|
|
||||||
|
request = new BulkRequest();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (request.numberOfActions() > 0) {
|
||||||
|
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||||
|
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||||
|
if (bulkResponse.hasFailures()) {
|
||||||
|
// 处理错误
|
||||||
|
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||||
|
} else {
|
||||||
|
processedCount += request.numberOfActions();
|
||||||
|
println "Processed " + processedCount + " documents"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user