update
This commit is contained in:
+110
@@ -0,0 +1,110 @@
|
||||
package com.fangjia.script.gj.village
|
||||
|
||||
import com.fangjia.script.gj.EsSearchUtils
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
DBCollection TOWN_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline")
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
|
||||
|
||||
def filePath = "D:\\数据\\涉农\\县.xlsx"
|
||||
ExcelUtils.multiThreadingModify(filePath, { row ->
|
||||
// if(!row['D']){
|
||||
// def rets = EsSearchUtils.findPois(row['B'], row['C'], row['C']+"人民政府")
|
||||
// if(rets){
|
||||
// def obj = findOneByName(rets, row['C']+"人民政府")
|
||||
// if(obj){
|
||||
// row['D'] = obj.lng
|
||||
// row['E'] = obj.lat
|
||||
// println row['C'] + " "+obj
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
|
||||
// if(row['D'] && !row['F']){
|
||||
// def townMap = findTownRange(row['B'], TOWN_POLYLINECOLL)
|
||||
// def map = ["lon":row['D'], "lat":row['E']]
|
||||
// def township = InsidePolygon.isInArea2(map.lon, map.lat, townMap)
|
||||
// row['F'] = township
|
||||
// }
|
||||
|
||||
if(row['F'] && !row['K']){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city_name", row['B'])
|
||||
q.put("area_name", row['C'])
|
||||
def streetName = row['I']
|
||||
q.put("street_name", streetName)
|
||||
def count = COMMUNITY_CONN.count(q);
|
||||
row['K'] = count
|
||||
// row['H'] = streetName
|
||||
}
|
||||
// def gaodeMap = getTownshipInfo(row['D']+","+row['E'])
|
||||
// if(gaodeMap && gaodeMap.township){
|
||||
// row['I'] = gaodeMap.township
|
||||
// }
|
||||
|
||||
})
|
||||
|
||||
|
||||
def getTownshipInfo(def glocation){
|
||||
def township = null
|
||||
def district = null
|
||||
def jsonObject = null
|
||||
def stop = false
|
||||
def count = 0
|
||||
while(!stop){
|
||||
if(glocation){
|
||||
jsonObject = GaoDeApiUtils.geocode(glocation)
|
||||
if(jsonObject && jsonObject.regeocode) {
|
||||
def addressComponent = jsonObject.regeocode.addressComponent
|
||||
if(addressComponent){
|
||||
township = addressComponent.township
|
||||
district = addressComponent.district
|
||||
}
|
||||
}
|
||||
}
|
||||
if(district || count>10){
|
||||
stop = true
|
||||
}else{
|
||||
count ++
|
||||
Thread.sleep(3000)
|
||||
print "."
|
||||
}
|
||||
}
|
||||
|
||||
def map = [:]
|
||||
map.district = district
|
||||
map.township = township
|
||||
return map;
|
||||
}
|
||||
|
||||
|
||||
def findOneByName(def rets, name){
|
||||
def obj = null
|
||||
for(def r : rets){
|
||||
if(r.name == name){
|
||||
obj = r
|
||||
break;
|
||||
}
|
||||
}
|
||||
return obj
|
||||
}
|
||||
|
||||
|
||||
def findTownRange(city, TOWN_POLYLINECOLL){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city",city)
|
||||
def dbcursor = TOWN_POLYLINECOLL.find(q)
|
||||
def townMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def pointList = InsidePolygon.createPoint3(d.polyline);
|
||||
townMap.put(d.fjw_region +"##"+ d.town, pointList)
|
||||
}
|
||||
return townMap
|
||||
}
|
||||
+53
-12
@@ -63,22 +63,59 @@ class SuggestionDataSynch extends BaseBatch{
|
||||
mcaDistMap = queryDistNameCity()
|
||||
|
||||
for (def c : citys) {
|
||||
def allRets = []
|
||||
index++
|
||||
// if(!["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"].contains(c)){
|
||||
updating(c, "district")
|
||||
updating(c, "office")
|
||||
updating(c, "shop")
|
||||
def ret1 = updating(c, "district")
|
||||
def ret2 = updating(c, "office")
|
||||
def ret3 = updating(c, "shop")
|
||||
|
||||
// }
|
||||
// if(["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"].contains(c)) {
|
||||
poiData(c)
|
||||
def ret4 = poiData(c)
|
||||
// }
|
||||
|
||||
println index + " " + c + " 完成"
|
||||
allRets.addAll(ret1)
|
||||
allRets.addAll(ret2)
|
||||
allRets.addAll(ret3)
|
||||
allRets.addAll(ret4)
|
||||
|
||||
println index + " " + c + " 完成 "+ allRets.size()
|
||||
saveToMG(c, allRets)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
def saveToMG(city, rows){
|
||||
|
||||
def data = fullCityMap.get(city)
|
||||
def q = new BasicDBObject("city", data.city)
|
||||
def count = SUGGESTION_CONN.count(q)
|
||||
if(count>0){
|
||||
SUGGESTION_CONN.remove(q)
|
||||
}
|
||||
|
||||
def rets = []
|
||||
for(def i=0;i<rows.size();i++){
|
||||
def m = rows[i]
|
||||
def obj = new BasicDBObject()
|
||||
m.each{k,v->
|
||||
obj.put(k,v)
|
||||
}
|
||||
rets << obj
|
||||
if(i%50000==0){
|
||||
SUGGESTION_CONN.insert(rets)
|
||||
rets.clear()
|
||||
}
|
||||
}
|
||||
if(rets.size()>0){
|
||||
SUGGESTION_CONN.insert(rets)
|
||||
rets.clear()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def updating(city, cat){
|
||||
def dists = CommonDatautils.findAllDistrictsFromMg(city, cat, ["city","region","name","address","mapping_alias","addr_alias","alias","cat","estate_type2","lng","lat","township"])
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
@@ -98,7 +135,8 @@ class SuggestionDataSynch extends BaseBatch{
|
||||
|
||||
def map = createData(dist)
|
||||
//往100 上同步
|
||||
updateData(map)
|
||||
// updateData(map)
|
||||
rets << map
|
||||
println city +" "+cat +" "+totalCount.get() +"/"+dists.size()+" "+dist.name+" "+map.id
|
||||
}
|
||||
} catch (Exception e) {
|
||||
@@ -140,7 +178,7 @@ class SuggestionDataSynch extends BaseBatch{
|
||||
map.alias = dist.alias
|
||||
}
|
||||
if(dist.addr_alias && dist.addr_alias.size()>0){
|
||||
map.addr_alias = dist.addr_alias
|
||||
map.addr_alias = dist.addr_alias
|
||||
}
|
||||
map.type = dist.cat
|
||||
if(dist.lng){
|
||||
@@ -183,9 +221,11 @@ class SuggestionDataSynch extends BaseBatch{
|
||||
def data = fullCityMap.get(city)
|
||||
if(data){
|
||||
def results = CommonDatautils.findAmapDetailBySql(data.city)
|
||||
createPoi(results, data, townMap)
|
||||
def rets = createPoi(results, data, townMap)
|
||||
println city +" 完成!"
|
||||
return rets;
|
||||
}
|
||||
return []
|
||||
|
||||
}
|
||||
|
||||
@@ -227,7 +267,8 @@ class SuggestionDataSynch extends BaseBatch{
|
||||
}
|
||||
println map.township
|
||||
//往100 上同步
|
||||
updateData(map)
|
||||
// updateData(map)
|
||||
rets << map
|
||||
println dist.city +" "+totalCount.get() +"/"+results.size()+" "+dist.name+" "+map.id
|
||||
}
|
||||
} catch (Exception e) {
|
||||
@@ -256,13 +297,13 @@ class SuggestionDataSynch extends BaseBatch{
|
||||
if(d.city == "市辖区"){
|
||||
fjwcity = RegionData.formatCity(d.province)
|
||||
if(fjwcity){
|
||||
d.city = d.province
|
||||
d.city = d.province
|
||||
}
|
||||
}
|
||||
}
|
||||
}else if(d.level == "区域"){
|
||||
if(d.administer == "省直辖县级行政区划"){
|
||||
fjwcity = RegionData.formatCity(d.city)
|
||||
fjwcity = RegionData.formatCity(d.city)
|
||||
if(fjwcity){
|
||||
d.city = d.district
|
||||
}
|
||||
@@ -290,7 +331,7 @@ class SuggestionDataSynch extends BaseBatch{
|
||||
}else if(d.city == "市辖区"){
|
||||
def fjwcity = RegionData.formatCity(d.province)
|
||||
if(fjwcity){
|
||||
def fjwdist = RegionData.formatRegion(fjwcity, d.district)
|
||||
def fjwdist = RegionData.formatRegion(fjwcity, d.district)
|
||||
retMap.put(fjwcity+"#"+fjwdist, d)
|
||||
}
|
||||
}else{
|
||||
|
||||
Reference in New Issue
Block a user