From 5419c4b90e8b892b553ceaf5cec640220eeaf9f8 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 8 Dec 2023 17:08:52 +0800 Subject: [PATCH] update --- .../BigdataCommunityBatch.groovy | 6 +- .../gj/xiamen/GaoDePoiDataProcess.groovy | 27 +++++++- .../fangjia/script/gj/xiamen/PoiWash.groovy | 14 ++-- .../script/gj/xiamen/SeaweedDataSynch.groovy | 66 ++++++++++++++++++- 4 files changed, 101 insertions(+), 12 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index 84782eb55..0ea40b8c2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -90,14 +90,16 @@ class BigdataCommunityBatch extends BaseBatch{ if(today > 10){ delPgData() saveToPgSql(rows) + + //更新静态数据表 + updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28") } // ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据") // def datas = readTxt("d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat") // delPgData() // saveToPgSql2(datas) - //更新静态数据表 - updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28") + } def addDeleteDist(dists, ccbMaps, rets){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy index c09bd3843..755c9b7eb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy @@ -9,6 +9,7 @@ import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.looplineinfo.InsidePolygon +import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils @@ -49,8 +50,9 @@ class GaoDePoiDataProcess extends BaseBatch{ def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; - DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi_all_8", MAP_POI_URL) + DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MAP_POI_URL) + DBCollection MAPPOI_CONN_TEST = MongoUtils.getCollectionByName2("fangjia", "map_poi_all_8", MAP_POI_URL) static DBCollection TOWN_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline") @@ -67,6 +69,29 @@ class GaoDePoiDataProcess extends BaseBatch{ // poiCatProcess() // addressAndCatProcess() // synchNetwork() +// println DateUtils.parseFromTightByPattern("2023-11-30","yyyy-MM-dd").getTime() +// sysnchPoi() + } + + def sysnchPoi(){ + def q = new BasicDBObject() + q.put("version", new BasicDBObject('$gte',1701273600000)) + def dbcursor = MAPPOI_CONN_TEST.find(q).batchSize(20000) + def ret = [] + def i = 0 + for(def d : dbcursor){ + i++ + ret << d + if(ret.size()%50000==0){ + println "第 "+(i/50000)+" 个"+ret.size() + MAPPOI_CONN.insert(ret); + ret = []; + } + } + + if(ret.size()>0){ + MAPPOI_CONN.insert(ret); + } } def synchCompany(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/PoiWash.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/PoiWash.groovy index ab238f42c..302cd02e2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/PoiWash.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/PoiWash.groovy @@ -40,10 +40,10 @@ class PoiWash { // task.run() // task.synch() // task.check() -// task.deduplicate() + task.deduplicate() // task.updateEsMapPoi() // task.checkSpecialSymbols() - task.splitRoad() +// task.splitRoad() } def updateId(){ @@ -160,7 +160,7 @@ class PoiWash { def run(){ //"上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市" //"重庆市","三明市","宁德市","泉州市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市" - def citys = ["上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市"] + def citys = ["杭州市","宁波市","温州市","嘉兴市","台州市","湖州市","绍兴市","金华市","舟山市","衢州市","丽水市"] for(def city : citys){ townsMap = findAllTownship(city) regions = findAllRegions(city) @@ -401,8 +401,8 @@ class PoiWash { def deduplicate(){ - //"三明市", - def citys = ["三明市", "漳州市","宁德市","泉州市","厦门市","福州市","南平市","龙岩市","莆田市","上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市","重庆市"] + //"三明市","漳州市","宁德市","泉州市","厦门市","福州市","南平市","龙岩市","莆田市","上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市","重庆市" + def citys = ["杭州市","宁波市","温州市","嘉兴市","台州市","湖州市","绍兴市","金华市","舟山市","衢州市","丽水市"] def total = 0 def upindex = 0 for(def city : citys){ @@ -563,7 +563,7 @@ class PoiWash { def check(){ // "漳州市","三明市","宁德市","泉州市","厦门市","福州市","南平市","龙岩市","莆田市","重庆市","深圳市","北京市","广州市","杭州市","苏州市","成都市", - def citys = ["重庆市"] + def citys = ["杭州市","宁波市","温州市","嘉兴市","台州市","湖州市","绍兴市","金华市","舟山市","衢州市","丽水市"] for(def city : citys){ def stop = false while(!stop){ @@ -651,7 +651,7 @@ class PoiWash { if(udata){ updateTotalCount.addAndGet(1) - udata.put("test", 2) +// udata.put("test", 2) MAPPOI_CONN.update(new BasicDBObject('id',map.id),new BasicDBObject('$set', udata)) println updateTotalCount.get()+" / "+totalCount.get()+" "+count +" "+city } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy index 774663ff3..50e68d9e9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy @@ -28,6 +28,10 @@ class SeaweedDataSynch extends BaseBatch{ DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL) DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL) + + DBCollection stat_administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "stat_administrative_division", MONGO100_URL) + public static final DBCollection stat_administrative_division_206_coll = MongoUtils.getCollectionByName("crawl", "stats_administrative_division", "192.168.0.206", 27020) + public static def provinceMap = CommonDatautils.findAllCityProvince() def lastMonthDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); @@ -39,6 +43,10 @@ class SeaweedDataSynch extends BaseBatch{ public static final DBCollection mca_administrative_division_coll = MongoUtils.getCollectionByName("crawl", "mca_administrative_division_2022", "192.168.0.206", 27020) + DBCollection community_100_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL) + + + def fullCityMap = null def mcaDistMap = null @@ -50,12 +58,66 @@ class SeaweedDataSynch extends BaseBatch{ public void process(Object args) { - run() - println "全国district, shop, office同步完成" +// run() +// println "全国district, shop, office同步完成" //民政部编码上传 //codeRun() //println "民政部省,市,区同步完成" //println new ObjectId("622c433afb459ab08fdab460").getDate() + + addTest() + } + + def addTest(){ + def q = new BasicDBObject() + def ret = [] + def dbcurosr = community_100_CONN.find(q).batchSize(20000) + for(def d : dbcurosr){ + def m = new BasicDBObject() + m.put("province" , d.province_name) + m.put("province_code", d.province_code) + if(['北京市','上海市','重庆市','天津市'].contains(d.province_name)){ + m.put("city" , d.province_name) + }else if(d.area_name=="自治区直辖县级行政区划" || d.area_name=="省直辖县级行政区划"){ + m.put("city" , d.area_name) + }else{ + m.put("city" , d.city_name) + } + m.put("city_code" ,d.city_code) + m.put("district" , d.area_name) + m.put("district_code" , d.area_code) + m.put("town" , d.street_name) + m.put("town_code" , d.street_code) + m.put("committee" , d.name) + m.put("committee_code" , d.code) + m.put("location",d.location) + m.put("type_code" ,d.type) + if(d.type=="111"){ + m.put("type" , "主城区") + }else if(d.type=="112"){ + m.put("type" , "城乡结合区") + }else if(d.type=="121"){ + m.put("type" , "镇中心区") + }else if(d.type=="122"){ + m.put("type" , "镇乡结合区") + }else if(d.type=="123"){ + m.put("type" , "特殊区域") + }else if(d.type=="210"){ + m.put("type" , "乡中心区") + }else if(d.type=="220"){ + m.put("type" , "村庄") + } + + ret << m + + if(ret.size()%50000==0){ + stat_administrative_division_100_CONN.insert(ret) + ret = [] + } + } + if(ret.size()>0){ + stat_administrative_division_100_CONN.insert(ret) + } }