+4
-2
@@ -90,14 +90,16 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
if(today > 10){
|
||||
delPgData()
|
||||
saveToPgSql(rows)
|
||||
|
||||
//更新静态数据表
|
||||
updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28")
|
||||
}
|
||||
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据")
|
||||
// def datas = readTxt("d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat")
|
||||
// delPgData()
|
||||
// saveToPgSql2(datas)
|
||||
|
||||
//更新静态数据表
|
||||
updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28")
|
||||
|
||||
}
|
||||
|
||||
def addDeleteDist(dists, ccbMaps, rets){
|
||||
|
||||
+26
-1
@@ -9,6 +9,7 @@ import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
@@ -49,8 +50,9 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
|
||||
|
||||
def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi_all_8", MAP_POI_URL)
|
||||
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MAP_POI_URL)
|
||||
|
||||
DBCollection MAPPOI_CONN_TEST = MongoUtils.getCollectionByName2("fangjia", "map_poi_all_8", MAP_POI_URL)
|
||||
|
||||
static DBCollection TOWN_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline")
|
||||
|
||||
@@ -67,6 +69,29 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
// poiCatProcess()
|
||||
// addressAndCatProcess()
|
||||
// synchNetwork()
|
||||
// println DateUtils.parseFromTightByPattern("2023-11-30","yyyy-MM-dd").getTime()
|
||||
// sysnchPoi()
|
||||
}
|
||||
|
||||
def sysnchPoi(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("version", new BasicDBObject('$gte',1701273600000))
|
||||
def dbcursor = MAPPOI_CONN_TEST.find(q).batchSize(20000)
|
||||
def ret = []
|
||||
def i = 0
|
||||
for(def d : dbcursor){
|
||||
i++
|
||||
ret << d
|
||||
if(ret.size()%50000==0){
|
||||
println "第 "+(i/50000)+" 个"+ret.size()
|
||||
MAPPOI_CONN.insert(ret);
|
||||
ret = [];
|
||||
}
|
||||
}
|
||||
|
||||
if(ret.size()>0){
|
||||
MAPPOI_CONN.insert(ret);
|
||||
}
|
||||
}
|
||||
|
||||
def synchCompany(){
|
||||
|
||||
@@ -40,10 +40,10 @@ class PoiWash {
|
||||
// task.run()
|
||||
// task.synch()
|
||||
// task.check()
|
||||
// task.deduplicate()
|
||||
task.deduplicate()
|
||||
// task.updateEsMapPoi()
|
||||
// task.checkSpecialSymbols()
|
||||
task.splitRoad()
|
||||
// task.splitRoad()
|
||||
}
|
||||
|
||||
def updateId(){
|
||||
@@ -160,7 +160,7 @@ class PoiWash {
|
||||
def run(){
|
||||
//"上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市"
|
||||
//"重庆市","三明市","宁德市","泉州市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市"
|
||||
def citys = ["上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市"]
|
||||
def citys = ["杭州市","宁波市","温州市","嘉兴市","台州市","湖州市","绍兴市","金华市","舟山市","衢州市","丽水市"]
|
||||
for(def city : citys){
|
||||
townsMap = findAllTownship(city)
|
||||
regions = findAllRegions(city)
|
||||
@@ -401,8 +401,8 @@ class PoiWash {
|
||||
|
||||
|
||||
def deduplicate(){
|
||||
//"三明市",
|
||||
def citys = ["三明市", "漳州市","宁德市","泉州市","厦门市","福州市","南平市","龙岩市","莆田市","上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市","重庆市"]
|
||||
//"三明市","漳州市","宁德市","泉州市","厦门市","福州市","南平市","龙岩市","莆田市","上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市","重庆市"
|
||||
def citys = ["杭州市","宁波市","温州市","嘉兴市","台州市","湖州市","绍兴市","金华市","舟山市","衢州市","丽水市"]
|
||||
def total = 0
|
||||
def upindex = 0
|
||||
for(def city : citys){
|
||||
@@ -563,7 +563,7 @@ class PoiWash {
|
||||
|
||||
def check(){
|
||||
// "漳州市","三明市","宁德市","泉州市","厦门市","福州市","南平市","龙岩市","莆田市","重庆市","深圳市","北京市","广州市","杭州市","苏州市","成都市",
|
||||
def citys = ["重庆市"]
|
||||
def citys = ["杭州市","宁波市","温州市","嘉兴市","台州市","湖州市","绍兴市","金华市","舟山市","衢州市","丽水市"]
|
||||
for(def city : citys){
|
||||
def stop = false
|
||||
while(!stop){
|
||||
@@ -651,7 +651,7 @@ class PoiWash {
|
||||
|
||||
if(udata){
|
||||
updateTotalCount.addAndGet(1)
|
||||
udata.put("test", 2)
|
||||
// udata.put("test", 2)
|
||||
MAPPOI_CONN.update(new BasicDBObject('id',map.id),new BasicDBObject('$set', udata))
|
||||
println updateTotalCount.get()+" / "+totalCount.get()+" "+count +" "+city
|
||||
}
|
||||
|
||||
+64
-2
@@ -28,6 +28,10 @@ class SeaweedDataSynch extends BaseBatch{
|
||||
DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL)
|
||||
DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL)
|
||||
|
||||
|
||||
DBCollection stat_administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "stat_administrative_division", MONGO100_URL)
|
||||
public static final DBCollection stat_administrative_division_206_coll = MongoUtils.getCollectionByName("crawl", "stats_administrative_division", "192.168.0.206", 27020)
|
||||
|
||||
public static def provinceMap = CommonDatautils.findAllCityProvince()
|
||||
|
||||
def lastMonthDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
@@ -39,6 +43,10 @@ class SeaweedDataSynch extends BaseBatch{
|
||||
public static final DBCollection mca_administrative_division_coll = MongoUtils.getCollectionByName("crawl", "mca_administrative_division_2022", "192.168.0.206", 27020)
|
||||
|
||||
|
||||
DBCollection community_100_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
|
||||
|
||||
|
||||
|
||||
def fullCityMap = null
|
||||
def mcaDistMap = null
|
||||
|
||||
@@ -50,12 +58,66 @@ class SeaweedDataSynch extends BaseBatch{
|
||||
public void process(Object args) {
|
||||
|
||||
|
||||
run()
|
||||
println "全国district, shop, office同步完成"
|
||||
// run()
|
||||
// println "全国district, shop, office同步完成"
|
||||
//民政部编码上传
|
||||
//codeRun()
|
||||
//println "民政部省,市,区同步完成"
|
||||
//println new ObjectId("622c433afb459ab08fdab460").getDate()
|
||||
|
||||
addTest()
|
||||
}
|
||||
|
||||
def addTest(){
|
||||
def q = new BasicDBObject()
|
||||
def ret = []
|
||||
def dbcurosr = community_100_CONN.find(q).batchSize(20000)
|
||||
for(def d : dbcurosr){
|
||||
def m = new BasicDBObject()
|
||||
m.put("province" , d.province_name)
|
||||
m.put("province_code", d.province_code)
|
||||
if(['北京市','上海市','重庆市','天津市'].contains(d.province_name)){
|
||||
m.put("city" , d.province_name)
|
||||
}else if(d.area_name=="自治区直辖县级行政区划" || d.area_name=="省直辖县级行政区划"){
|
||||
m.put("city" , d.area_name)
|
||||
}else{
|
||||
m.put("city" , d.city_name)
|
||||
}
|
||||
m.put("city_code" ,d.city_code)
|
||||
m.put("district" , d.area_name)
|
||||
m.put("district_code" , d.area_code)
|
||||
m.put("town" , d.street_name)
|
||||
m.put("town_code" , d.street_code)
|
||||
m.put("committee" , d.name)
|
||||
m.put("committee_code" , d.code)
|
||||
m.put("location",d.location)
|
||||
m.put("type_code" ,d.type)
|
||||
if(d.type=="111"){
|
||||
m.put("type" , "主城区")
|
||||
}else if(d.type=="112"){
|
||||
m.put("type" , "城乡结合区")
|
||||
}else if(d.type=="121"){
|
||||
m.put("type" , "镇中心区")
|
||||
}else if(d.type=="122"){
|
||||
m.put("type" , "镇乡结合区")
|
||||
}else if(d.type=="123"){
|
||||
m.put("type" , "特殊区域")
|
||||
}else if(d.type=="210"){
|
||||
m.put("type" , "乡中心区")
|
||||
}else if(d.type=="220"){
|
||||
m.put("type" , "村庄")
|
||||
}
|
||||
|
||||
ret << m
|
||||
|
||||
if(ret.size()%50000==0){
|
||||
stat_administrative_division_100_CONN.insert(ret)
|
||||
ret = []
|
||||
}
|
||||
}
|
||||
if(ret.size()>0){
|
||||
stat_administrative_division_100_CONN.insert(ret)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user