diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy index a749b5541..fa7fa0ac7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/NewGaoDePoiBatch.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.gj.data_handle +import com.fangjia.core.util.GeoUtils import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher @@ -8,13 +9,19 @@ import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.gj.xiamen.ElasticUtils89 import com.fangjia.script.looplineinfo.InsidePolygon import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.ProvinceData +import com.mongodb.BasicDBObject +import org.apache.commons.io.FileUtils import org.elasticsearch.action.bulk.BulkRequest import org.elasticsearch.action.bulk.BulkResponse import org.elasticsearch.action.index.IndexRequest import org.elasticsearch.action.support.WriteRequest import org.elasticsearch.client.RequestOptions import org.elasticsearch.common.xcontent.XContentType +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders +import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService @@ -39,8 +46,8 @@ class NewGaoDePoiBatch extends BaseBatch{ @Override void process(Object args) { - run() -// oldPoiMagre() +// run() + oldPoiMagre() } def run(){ @@ -349,7 +356,164 @@ class NewGaoDePoiBatch extends BaseBatch{ } def oldPoiMagre(){ + def cityList = CommonDatautils.findFullNameCitys().reverse() + for(def cityObj : cityList){ +// if(!cityObj.city.contains("固原")){ +// continue +// } + def fileName = "/home/goojia/gj/poi/poi/${cityObj.city}.csv" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\merge_poi\\${cityObj.city}.csv" + } + File file=new File(fileName); + if(file.exists()){ + def results = readCSV(fileName) + println cityObj.province+" " +cityObj.city+" "+results.size() + if(results){ + def megerMap = poiRepeat(cityObj.city, results) + save(megerMap.values()) + } + } + } + } + + def poiRepeat(city, oldPois){ + def rets = new ConcurrentHashMap() + + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(oldPois.size()) + def townMap = CommonDatautils.queryTownByCity(city) + def esMap = findEsPoiByCity(city) + + println "Es 加载完成:"+new Date().format("yyyy-MM-dd HH:mm:ss") + for(def i=0;i + def district = "" + if(d.district){ + district = d.district + } + def regionMap = map.get(district) + if(!regionMap){ + regionMap = new HashMap() + } + def o = new BasicDBObject() + o.put("city", d.city) + o.put("district", d.district) + o.put("name", d.name) + o.put("township", d.township) + o.put("location", d.location) + regionMap.put(d.name.replace("-","").replace("·",""), o) + map.put(district, regionMap) + }) + return map + } + + + def readCSV(filename){ + def lines = FileUtils.readLines(new File(filename)) + def rets = []; + for(line in lines) { + if(line){ + String[] arrs = line.split("\\|@\\|") + def o = new BasicDBObject() + o.put("province",arrs[0]) + o.put("city" , arrs[1]) + o.put("district",arrs[2]) + o.put("name",arrs[3]) + o.put("township",arrs[4]) + o.put("address",arrs[5]) + o.put("cat",arrs[6]) + o.put("type",arrs[7]) + o.put("secondary_type",arrs[8]) + o.put("small_type" ,arrs[9]) + if(arrs[10]){ + o.put("data_id",arrs[10]) + }else{ + o.put("data_id",arrs[11]) + } + + o.put("spell_name",arrs[12]) + def location = [:] + if(arrs[13]){ + location.lon = arrs[13].toString().toDouble() + location.lat = arrs[14].toString().toDouble() + } + o.put("location",location) + o.put("data_source",arrs[15]) + o.put("status",arrs[16]) + o.put("version",arrs[17]) + rets << o + } + } + return rets; } }