Merge branch 'script_gj' of http://gitlab.fangjia.com/dev0/main into script_gj
This commit is contained in:
+11
-6
@@ -4858,6 +4858,7 @@ class CommonDatautils {
|
|||||||
keys.put("city",1)
|
keys.put("city",1)
|
||||||
keys.put("district",1)
|
keys.put("district",1)
|
||||||
keys.put("alias",1)
|
keys.put("alias",1)
|
||||||
|
keys.put("mzb_district_name",1)
|
||||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||||
def retMap = [:]
|
def retMap = [:]
|
||||||
for(def d : dbcursor){
|
for(def d : dbcursor){
|
||||||
@@ -4866,7 +4867,7 @@ class CommonDatautils {
|
|||||||
regionMap = [:]
|
regionMap = [:]
|
||||||
retMap.put(d.city, regionMap)
|
retMap.put(d.city, regionMap)
|
||||||
}
|
}
|
||||||
regionMap.put(d.district, d.alias)
|
regionMap.put(d.district, d)
|
||||||
}
|
}
|
||||||
|
|
||||||
return retMap
|
return retMap
|
||||||
@@ -4975,18 +4976,22 @@ class CommonDatautils {
|
|||||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||||
def townMap = [:]
|
def townMap = [:]
|
||||||
for(def d : dbcursor){
|
for(def d : dbcursor){
|
||||||
def list = townMap.get(d.district)
|
if(d.mzb_district_name){
|
||||||
if(!list){
|
def list = townMap.get(d.mzb_district_name)
|
||||||
list = []
|
if(!list){
|
||||||
|
list = []
|
||||||
|
}
|
||||||
|
list << d
|
||||||
|
townMap.put(d.mzb_district_name, list)
|
||||||
}
|
}
|
||||||
list << d
|
|
||||||
townMap.put(d.district, list)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
println "[${city}] 乡镇街道 加载完成 "+new Date().format("yyyy-MM-dd HH:mm:ss")
|
println "[${city}] 乡镇街道 加载完成 "+new Date().format("yyyy-MM-dd HH:mm:ss")
|
||||||
return townMap
|
return townMap
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
static def queryTownByCity(city){
|
static def queryTownByCity(city){
|
||||||
|
|
||||||
def q = new BasicDBObject()
|
def q = new BasicDBObject()
|
||||||
|
|||||||
+56
-11
@@ -10,7 +10,6 @@ import com.fangjia.script.util.ElasticUtils
|
|||||||
import com.fangjia.script.util.MongoUtils
|
import com.fangjia.script.util.MongoUtils
|
||||||
import com.mongodb.BasicDBObject
|
import com.mongodb.BasicDBObject
|
||||||
import com.mongodb.DBCollection
|
import com.mongodb.DBCollection
|
||||||
import com.rabbitmq.client.impl.AMQImpl
|
|
||||||
import groovy.json.JsonSlurper
|
import groovy.json.JsonSlurper
|
||||||
import net.sf.json.JSONObject
|
import net.sf.json.JSONObject
|
||||||
|
|
||||||
@@ -19,7 +18,6 @@ import java.util.concurrent.CountDownLatch
|
|||||||
import java.util.concurrent.ExecutorService
|
import java.util.concurrent.ExecutorService
|
||||||
import java.util.concurrent.Executors
|
import java.util.concurrent.Executors
|
||||||
import java.util.concurrent.atomic.AtomicLong
|
import java.util.concurrent.atomic.AtomicLong
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 添加高德地图,百度地图围栏
|
* 添加高德地图,百度地图围栏
|
||||||
*/
|
*/
|
||||||
@@ -32,6 +30,9 @@ class AddDistrictShapeBatch extends BaseBatch{
|
|||||||
DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
|
DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
|
||||||
public static final DBCollection seaweed_fence_coll = MongoUtils.getCollectionByName("fangjia_base", "seaweed_fence", "mongo.db.fangjia.com")
|
public static final DBCollection seaweed_fence_coll = MongoUtils.getCollectionByName("fangjia_base", "seaweed_fence", "mongo.db.fangjia.com")
|
||||||
|
|
||||||
|
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||||
|
DBCollection seaweed_fence100_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed_fence", MONGO100_URL)
|
||||||
|
|
||||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||||
|
|
||||||
static main(args) {
|
static main(args) {
|
||||||
@@ -41,17 +42,20 @@ class AddDistrictShapeBatch extends BaseBatch{
|
|||||||
@Override
|
@Override
|
||||||
public void process(Object args) {
|
public void process(Object args) {
|
||||||
|
|
||||||
// findGaoDeDatas2("苏州市","吴中区", "水岸清华瀚宫" )
|
// findGaoDeDatas2("苏州市","吴中区", "水岸清华瀚宫" )//测试
|
||||||
def citys = CommonDatautils.findAllCity()
|
// def citys = CommonDatautils.findAllCity()
|
||||||
def index = 1
|
// def index = 1
|
||||||
for(def c : citys){
|
// for(def c : citys){
|
||||||
println c +" "+index
|
// println c +" "+index
|
||||||
index ++
|
// index ++
|
||||||
def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "estate_type2"])
|
// def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "estate_type2"])
|
||||||
// baiduMatch(c, districts)
|
// baiduMatch(c, districts)
|
||||||
gaodeMatch(c, districts)
|
// gaodeMatch(c, districts)
|
||||||
|
//
|
||||||
|
// }
|
||||||
|
|
||||||
}
|
// checkRangeCount()
|
||||||
|
synch100()
|
||||||
}
|
}
|
||||||
|
|
||||||
def baiduMatch(city,districts){
|
def baiduMatch(city,districts){
|
||||||
@@ -373,4 +377,45 @@ class AddDistrictShapeBatch extends BaseBatch{
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def checkRangeCount(){
|
||||||
|
def dbcursor = seaweed_fence_coll.find(new BasicDBObject()).batchSize(100)
|
||||||
|
|
||||||
|
def count = 0
|
||||||
|
def index =0
|
||||||
|
for(def d : dbcursor){
|
||||||
|
index ++
|
||||||
|
if(d.location){
|
||||||
|
def coordinates = d.location.coordinates
|
||||||
|
if(coordinates.size()>1){
|
||||||
|
count++
|
||||||
|
println count +" "+ d.city +" "+d.name
|
||||||
|
}
|
||||||
|
}else{
|
||||||
|
println d.city + " "+d.name
|
||||||
|
}
|
||||||
|
|
||||||
|
if(index%1000==0){
|
||||||
|
println index
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
def synch100(){
|
||||||
|
def dbcursor = seaweed_fence_coll.find(new BasicDBObject()).batchSize(100)
|
||||||
|
def rets = []
|
||||||
|
for(def d : dbcursor) {
|
||||||
|
rets << d
|
||||||
|
if(rets.size()%1000==0){
|
||||||
|
seaweed_fence100_CONN.insert(rets)
|
||||||
|
rets.clear()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if(rets.size()>0){
|
||||||
|
seaweed_fence100_CONN.insert(rets)
|
||||||
|
rets.clear()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
+1
-1
@@ -37,7 +37,7 @@ class AutoAddDistrictBatch extends BaseBatch{
|
|||||||
@Override
|
@Override
|
||||||
void process(Object args) {
|
void process(Object args) {
|
||||||
//查找第三方匹配没有匹配上的小区
|
//查找第三方匹配没有匹配上的小区
|
||||||
def citys = CommonDatautils.findAllCity()
|
def citys = ["上海"]//CommonDatautils.findAllCity()
|
||||||
def list = []
|
def list = []
|
||||||
list << ["城市", "区域", "名称", "成交量", "房源量", "百度地图", "高德地图"]
|
list << ["城市", "区域", "名称", "成交量", "房源量", "百度地图", "高德地图"]
|
||||||
for(def c : citys){
|
for(def c : citys){
|
||||||
|
|||||||
+50
-30
@@ -23,6 +23,7 @@ class RegionStandardBatch extends BaseBatch{
|
|||||||
def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity()
|
def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity()
|
||||||
def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity()
|
def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity()
|
||||||
|
|
||||||
|
|
||||||
static main(args) {
|
static main(args) {
|
||||||
runBatch(new RegionStandardBatch(), args)
|
runBatch(new RegionStandardBatch(), args)
|
||||||
}
|
}
|
||||||
@@ -37,14 +38,15 @@ class RegionStandardBatch extends BaseBatch{
|
|||||||
seaweedRegionStandardByCat("district")
|
seaweedRegionStandardByCat("district")
|
||||||
}
|
}
|
||||||
def poiRegionStandard(){
|
def poiRegionStandard(){
|
||||||
|
|
||||||
def runCitys = ["重庆","三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德"]
|
def runCitys = ["重庆","三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德"]
|
||||||
def index = 0
|
def index = 0
|
||||||
def citys = CommonDatautils.findAllCity()
|
def citys = ["攀枝花"]//CommonDatautils.findAllCity()
|
||||||
for(def city : citys){
|
for(def city : citys){
|
||||||
index ++
|
index ++
|
||||||
if(index<32){
|
// if(index<32){
|
||||||
continue
|
// continue
|
||||||
}
|
// }
|
||||||
if(runCitys.contains(city)){
|
if(runCitys.contains(city)){
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -74,6 +76,7 @@ class RegionStandardBatch extends BaseBatch{
|
|||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -195,39 +198,56 @@ class RegionStandardBatch extends BaseBatch{
|
|||||||
def poiHaveRegionProcess(city, set, rMap, datas){
|
def poiHaveRegionProcess(city, set, rMap, datas){
|
||||||
def results = []
|
def results = []
|
||||||
for(def d : datas){
|
for(def d : datas){
|
||||||
if(set.contains(d.district)){
|
def tjjObj = rMap.get(d.district)
|
||||||
|
if(tjjObj && tjjObj.mzb_district_name && tjjObj.mzb_district_name==d.district){
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
def updateRegion = null
|
def updateRegion = null
|
||||||
for(def key : set){
|
def updateFlag = true
|
||||||
def alias = rMap.get(key)
|
if(tjjObj){
|
||||||
if(alias.contains(d.district)){
|
if(tjjObj.mzb_district_name && tjjObj.mzb_district_name!=d.district){
|
||||||
updateRegion = key
|
updateRegion = tjjObj.mzb_district_name
|
||||||
break;
|
|
||||||
}
|
}
|
||||||
}
|
|
||||||
if(updateRegion){
|
|
||||||
def m = [:]
|
|
||||||
m.put(d.id, ["district":updateRegion])
|
|
||||||
results << m
|
|
||||||
if(results.size()%100==0){
|
|
||||||
updateRegionByEs(results)
|
|
||||||
results.clear()
|
|
||||||
}
|
|
||||||
println d.city +" "+d.district+" "+d.name +" "+d.status+" 区域修改为:"+updateRegion
|
|
||||||
|
|
||||||
}else{
|
}else{
|
||||||
def m = [:]
|
for(def key : set){
|
||||||
m.put(d.id, ["status":1])
|
def tempObj = rMap.get(key)
|
||||||
results << m
|
def alias = tempObj.alias
|
||||||
if(results.size()%100==0){
|
if(alias.contains(d.district) || d.district==tempObj.mzb_district_name){
|
||||||
updateRegionByEs(results)
|
if(d.district!=tempObj.mzb_district_name){
|
||||||
results.clear()
|
updateRegion = key
|
||||||
|
}else{
|
||||||
|
updateFlag = false
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
println d.city +" "+d.district+" "+d.name +" "+d.status+" 状态修改为:删除"
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if(updateFlag){
|
||||||
|
if(updateRegion){
|
||||||
|
def m = [:]
|
||||||
|
m.put(d.id, ["district":updateRegion])
|
||||||
|
results << m
|
||||||
|
if(results.size()%100==0){
|
||||||
|
updateRegionByEs(results)
|
||||||
|
results.clear()
|
||||||
|
}
|
||||||
|
|
||||||
|
println d.city +" "+d.district+" "+d.name +" "+d.status+" 区域修改为:"+updateRegion
|
||||||
|
|
||||||
|
}else{
|
||||||
|
def m = [:]
|
||||||
|
m.put(d.id, ["status":1])
|
||||||
|
results << m
|
||||||
|
if(results.size()%100==0){
|
||||||
|
updateRegionByEs(results)
|
||||||
|
results.clear()
|
||||||
|
}
|
||||||
|
|
||||||
|
println d.city +" "+d.district+" "+d.name +" "+d.status+" 状态修改为:删除"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if(results.size()>0){
|
if(results.size()>0){
|
||||||
|
|||||||
+63
-23
@@ -1,17 +1,17 @@
|
|||||||
package com.fangjia.script.gj.data_handle
|
package com.fangjia.script.gj.data_handle
|
||||||
|
|
||||||
|
|
||||||
import com.fangjia.script.BaseBatch
|
import com.fangjia.script.BaseBatch
|
||||||
import com.fangjia.script.commData.CommonDatautils
|
import com.fangjia.script.commData.CommonDatautils
|
||||||
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
||||||
|
import com.fangjia.script.util.ExcelUtils
|
||||||
import com.mongodb.BasicDBObject
|
import com.mongodb.BasicDBObject
|
||||||
|
import org.apache.commons.io.FileUtils
|
||||||
import org.elasticsearch.action.bulk.BulkRequest
|
import org.elasticsearch.action.bulk.BulkRequest
|
||||||
import org.elasticsearch.action.update.UpdateRequest
|
import org.elasticsearch.action.update.UpdateRequest
|
||||||
import org.elasticsearch.client.RequestOptions
|
import org.elasticsearch.client.RequestOptions
|
||||||
import org.elasticsearch.common.xcontent.XContentType
|
import org.elasticsearch.common.xcontent.XContentType
|
||||||
import org.elasticsearch.index.query.QueryBuilder
|
import org.elasticsearch.index.query.QueryBuilder
|
||||||
import org.elasticsearch.index.query.QueryBuilders
|
import org.elasticsearch.index.query.QueryBuilders
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 区域标准化
|
* 区域标准化
|
||||||
* 1.小区库乡镇街道标准化,包含小区,商铺,写字楼
|
* 1.小区库乡镇街道标准化,包含小区,商铺,写字楼
|
||||||
@@ -23,12 +23,7 @@ class TownStandardBatch extends BaseBatch{
|
|||||||
def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity()
|
def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity()
|
||||||
def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity()
|
def regionFullNameMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionFullNameByCity()
|
||||||
|
|
||||||
def poiEsUrl = "http://192.168.0.89:9200/didata_map_poi/_search"
|
def updateList = []
|
||||||
|
|
||||||
def elasticsearchUrl = "http://192.168.0.89:9200/_search/scroll"
|
|
||||||
def initialSearchUrl = "http://192.168.0.89:9200/didata_map_poi/_search?scroll=1m"
|
|
||||||
def scrollTime = "1m" // Scroll context time-to-live
|
|
||||||
def pageSize = 1000 // Number of hits to return per scroll
|
|
||||||
|
|
||||||
static main(args) {
|
static main(args) {
|
||||||
runBatch(new TownStandardBatch(), args)
|
runBatch(new TownStandardBatch(), args)
|
||||||
@@ -70,7 +65,8 @@ class TownStandardBatch extends BaseBatch{
|
|||||||
}
|
}
|
||||||
|
|
||||||
def townStandardByPoi(){
|
def townStandardByPoi(){
|
||||||
def citys = ["重庆","龙岩","南平","漳州","福州","厦门","泉州","莆田","三明","宁德"]//CommonDatautils.findAllCity()
|
updateList << ["数据ID", "城市","区域","乡镇街道","名称","状态", "修改"]
|
||||||
|
def citys = CommonDatautils.findAllCity()
|
||||||
for(def city : citys){
|
for(def city : citys){
|
||||||
|
|
||||||
def data = fullCityMap.get(city)
|
def data = fullCityMap.get(city)
|
||||||
@@ -79,22 +75,33 @@ class TownStandardBatch extends BaseBatch{
|
|||||||
fullCityName = data.city
|
fullCityName = data.city
|
||||||
}
|
}
|
||||||
if(fullCityName){
|
if(fullCityName){
|
||||||
def townMap = CommonDatautils.queryTownByCity(fullCityName, ["city","district", "street", "alias"])
|
|
||||||
def rMap = regionFullNameMap.get(fullCityName)
|
def fileName = "/home/goojia/gj/poi/${fullCityName}.csv"
|
||||||
if(rMap){
|
if (CommonDatautils.isWindow()) {
|
||||||
def set = rMap.keySet()
|
fileName = "D:\\数据\\${fullCityName}.csv"
|
||||||
def rets = findEsPoiByCity(fullCityName)
|
}
|
||||||
println city +" "+rets.size()
|
File file=new File(fileName);
|
||||||
if(set.size()>1){
|
if(file.exists()){
|
||||||
poiHaveRegionProcess(fullCityName, townMap, rets)
|
def rMap = regionFullNameMap.get(fullCityName)
|
||||||
}else{
|
if(rMap){
|
||||||
|
def mzbRegionMap = CommonDatautils.queryMzbRegion(fullCityName)
|
||||||
|
def townMap = CommonDatautils.queryTownByCity(fullCityName, ["city","district","mzb_district_name", "street", "alias"])
|
||||||
|
def set = rMap.keySet()
|
||||||
|
// def rets = findEsPoiByCity(fullCityName)
|
||||||
|
def rets = readPoiCsv(fullCityName)
|
||||||
|
println city +" "+rets.size()
|
||||||
|
if(set.size()>1){
|
||||||
|
poiHaveRegionProcess(fullCityName, townMap, rets, mzbRegionMap)
|
||||||
|
}else{
|
||||||
// haveStreetProcess(city, set, cat)
|
// haveStreetProcess(city, set, cat)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}else{
|
}else{
|
||||||
println "没找到相应的全称:"+city
|
println "没找到相应的全称:"+city
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
ExcelUtils.write("D:\\数据\\乡镇街道待查数据.xlsx", updateList, "数据")
|
||||||
}
|
}
|
||||||
def haveRegionProcess(city, fullCityName, townMap, cat, shortRegionNameMap){
|
def haveRegionProcess(city, fullCityName, townMap, cat, shortRegionNameMap){
|
||||||
def results = []
|
def results = []
|
||||||
@@ -149,12 +156,15 @@ class TownStandardBatch extends BaseBatch{
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
def poiHaveRegionProcess(fullCityName, townMap, rets){
|
def poiHaveRegionProcess(fullCityName, townMap, rets, mzbRegionMap){
|
||||||
def results = []
|
def results = []
|
||||||
if(fullCityName){
|
if(fullCityName){
|
||||||
for(def d : rets){
|
for(def d : rets){
|
||||||
|
|
||||||
if(d.township){
|
if(d.township){
|
||||||
def towns = null
|
def towns = null
|
||||||
def updateTown = null
|
def updateTown = null
|
||||||
@@ -163,6 +173,8 @@ class TownStandardBatch extends BaseBatch{
|
|||||||
if(townDatas){
|
if(townDatas){
|
||||||
towns = townDatas
|
towns = townDatas
|
||||||
break
|
break
|
||||||
|
}else{
|
||||||
|
println(d.city +" "+d.district+" 没找下辖街道")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -193,6 +205,7 @@ class TownStandardBatch extends BaseBatch{
|
|||||||
// updateByEs(results)
|
// updateByEs(results)
|
||||||
// results.clear()
|
// results.clear()
|
||||||
// }
|
// }
|
||||||
|
updateList << [d.id, d.city, d.district, d.name,d.township, d.status, updateTown]
|
||||||
println d.city +" "+d.district+" "+d.name +" "+d.township+" 乡镇修改为:"+updateTown
|
println d.city +" "+d.district+" "+d.name +" "+d.township+" 乡镇修改为:"+updateTown
|
||||||
}else{
|
}else{
|
||||||
def m = [:]
|
def m = [:]
|
||||||
@@ -202,6 +215,7 @@ class TownStandardBatch extends BaseBatch{
|
|||||||
// updateByEs(results)
|
// updateByEs(results)
|
||||||
// results.clear()
|
// results.clear()
|
||||||
// }
|
// }
|
||||||
|
updateList << [d.id, d.city, d.district, d.name, d.township, d.status, ""]
|
||||||
println d.city +" "+d.district+" "+d.name +" "+d.township+" 状态修改为:空值"
|
println d.city +" "+d.district+" "+d.name +" "+d.township+" 状态修改为:空值"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -210,10 +224,10 @@ class TownStandardBatch extends BaseBatch{
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if(results.size()>0){
|
// if(results.size()>0){
|
||||||
updateByEs(results)
|
// updateByEs(results)
|
||||||
results.clear()
|
// results.clear()
|
||||||
}
|
// }
|
||||||
println fullCityName +" 处理完成"
|
println fullCityName +" 处理完成"
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -276,4 +290,30 @@ class TownStandardBatch extends BaseBatch{
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def readPoiCsv(filename){
|
||||||
|
def datas = readCSV(filename)
|
||||||
|
return datas
|
||||||
|
}
|
||||||
|
|
||||||
|
def readCSV(filename){
|
||||||
|
def lines = FileUtils.readLines(new File(filename))
|
||||||
|
def rets = [];
|
||||||
|
for(line in lines) {
|
||||||
|
if(line){
|
||||||
|
String[] arrs = line.split("\\|@\\|")
|
||||||
|
def o = new BasicDBObject()
|
||||||
|
o.put("city", arrs[0])
|
||||||
|
o.put("district", arrs[1])
|
||||||
|
o.put("name", arrs[2])
|
||||||
|
o.put("township", arrs[3])
|
||||||
|
o.put("status", arrs[4])
|
||||||
|
o.put("id", arrs[5])
|
||||||
|
rets << o
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
return rets;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user