Merge branch 'script_gj' into 'master'

update

See merge request !761
This commit is contained in:
2023-12-08 17:14:25 +08:00
4 changed files with 101 additions and 12 deletions
@@ -90,14 +90,16 @@ class BigdataCommunityBatch extends BaseBatch{
if(today > 10){
delPgData()
saveToPgSql(rows)
//更新静态数据表
updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28")
}
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据")
// def datas = readTxt("d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat")
// delPgData()
// saveToPgSql2(datas)
//更新静态数据表
updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28")
}
def addDeleteDist(dists, ccbMaps, rets){
@@ -9,6 +9,7 @@ import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
@@ -49,8 +50,9 @@ class GaoDePoiDataProcess extends BaseBatch{
def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi_all_8", MAP_POI_URL)
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MAP_POI_URL)
DBCollection MAPPOI_CONN_TEST = MongoUtils.getCollectionByName2("fangjia", "map_poi_all_8", MAP_POI_URL)
static DBCollection TOWN_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline")
@@ -67,6 +69,29 @@ class GaoDePoiDataProcess extends BaseBatch{
// poiCatProcess()
// addressAndCatProcess()
// synchNetwork()
// println DateUtils.parseFromTightByPattern("2023-11-30","yyyy-MM-dd").getTime()
// sysnchPoi()
}
def sysnchPoi(){
def q = new BasicDBObject()
q.put("version", new BasicDBObject('$gte',1701273600000))
def dbcursor = MAPPOI_CONN_TEST.find(q).batchSize(20000)
def ret = []
def i = 0
for(def d : dbcursor){
i++
ret << d
if(ret.size()%50000==0){
println "第 "+(i/50000)+" 个"+ret.size()
MAPPOI_CONN.insert(ret);
ret = [];
}
}
if(ret.size()>0){
MAPPOI_CONN.insert(ret);
}
}
def synchCompany(){
@@ -40,10 +40,10 @@ class PoiWash {
// task.run()
// task.synch()
// task.check()
// task.deduplicate()
task.deduplicate()
// task.updateEsMapPoi()
// task.checkSpecialSymbols()
task.splitRoad()
// task.splitRoad()
}
def updateId(){
@@ -160,7 +160,7 @@ class PoiWash {
def run(){
//"上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市"
//"重庆市","三明市","宁德市","泉州市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市"
def citys = ["上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市"]
def citys = ["杭州市","宁波市","温州市","嘉兴市","台州市","湖州市","绍兴市","金华市","舟山市","衢州市","丽水市"]
for(def city : citys){
townsMap = findAllTownship(city)
regions = findAllRegions(city)
@@ -401,8 +401,8 @@ class PoiWash {
def deduplicate(){
//"三明市",
def citys = ["三明市", "漳州市","宁德市","泉州市","厦门市","福州市","南平市","龙岩市","莆田市","上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市","重庆市"]
//"三明市","漳州市","宁德市","泉州市","厦门市","福州市","南平市","龙岩市","莆田市","上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市","重庆市"
def citys = ["杭州市","宁波市","温州市","嘉兴市","台州市","湖州市","绍兴市","金华市","舟山市","衢州市","丽水市"]
def total = 0
def upindex = 0
for(def city : citys){
@@ -563,7 +563,7 @@ class PoiWash {
def check(){
// "漳州市","三明市","宁德市","泉州市","厦门市","福州市","南平市","龙岩市","莆田市","重庆市","深圳市","北京市","广州市","杭州市","苏州市","成都市",
def citys = ["重庆市"]
def citys = ["杭州市","宁波市","温州市","嘉兴市","台州市","湖州市","绍兴市","金华市","舟山市","衢州市","丽水市"]
for(def city : citys){
def stop = false
while(!stop){
@@ -651,7 +651,7 @@ class PoiWash {
if(udata){
updateTotalCount.addAndGet(1)
udata.put("test", 2)
// udata.put("test", 2)
MAPPOI_CONN.update(new BasicDBObject('id',map.id),new BasicDBObject('$set', udata))
println updateTotalCount.get()+" / "+totalCount.get()+" "+count +" "+city
}
@@ -28,6 +28,10 @@ class SeaweedDataSynch extends BaseBatch{
DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL)
DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL)
DBCollection stat_administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "stat_administrative_division", MONGO100_URL)
public static final DBCollection stat_administrative_division_206_coll = MongoUtils.getCollectionByName("crawl", "stats_administrative_division", "192.168.0.206", 27020)
public static def provinceMap = CommonDatautils.findAllCityProvince()
def lastMonthDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
@@ -39,6 +43,10 @@ class SeaweedDataSynch extends BaseBatch{
public static final DBCollection mca_administrative_division_coll = MongoUtils.getCollectionByName("crawl", "mca_administrative_division_2022", "192.168.0.206", 27020)
DBCollection community_100_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
def fullCityMap = null
def mcaDistMap = null
@@ -50,12 +58,66 @@ class SeaweedDataSynch extends BaseBatch{
public void process(Object args) {
run()
println "全国district, shop, office同步完成"
// run()
// println "全国district, shop, office同步完成"
//民政部编码上传
//codeRun()
//println "民政部省,市,区同步完成"
//println new ObjectId("622c433afb459ab08fdab460").getDate()
addTest()
}
def addTest(){
def q = new BasicDBObject()
def ret = []
def dbcurosr = community_100_CONN.find(q).batchSize(20000)
for(def d : dbcurosr){
def m = new BasicDBObject()
m.put("province" , d.province_name)
m.put("province_code", d.province_code)
if(['北京市','上海市','重庆市','天津市'].contains(d.province_name)){
m.put("city" , d.province_name)
}else if(d.area_name=="自治区直辖县级行政区划" || d.area_name=="省直辖县级行政区划"){
m.put("city" , d.area_name)
}else{
m.put("city" , d.city_name)
}
m.put("city_code" ,d.city_code)
m.put("district" , d.area_name)
m.put("district_code" , d.area_code)
m.put("town" , d.street_name)
m.put("town_code" , d.street_code)
m.put("committee" , d.name)
m.put("committee_code" , d.code)
m.put("location",d.location)
m.put("type_code" ,d.type)
if(d.type=="111"){
m.put("type" , "主城区")
}else if(d.type=="112"){
m.put("type" , "城乡结合区")
}else if(d.type=="121"){
m.put("type" , "镇中心区")
}else if(d.type=="122"){
m.put("type" , "镇乡结合区")
}else if(d.type=="123"){
m.put("type" , "特殊区域")
}else if(d.type=="210"){
m.put("type" , "乡中心区")
}else if(d.type=="220"){
m.put("type" , "村庄")
}
ret << m
if(ret.size()%50000==0){
stat_administrative_division_100_CONN.insert(ret)
ret = []
}
}
if(ret.size()>0){
stat_administrative_division_100_CONN.insert(ret)
}
}