加日期

This commit is contained in:
2024-05-07 16:10:49 +08:00
parent 3a8b5254c3
commit 1874268b33
4 changed files with 267 additions and 35 deletions
@@ -159,7 +159,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMUNITY_FJW.dat"
def fileName = "/ccb/${curDate}/MODEL_000000_000000_BIGDATA_COMMUNITY_FJW.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMUNITY_FJW.dat"
}
@@ -246,7 +246,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMBUILDSTRU_FJW.dat"
def fileName = "/ccb/${curDate}/MODEL_000000_000000_BIGDATA_COMMBUILDSTRU_FJW.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMBUILDSTRU_FJW.dat"
}
@@ -304,7 +304,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMHOUSETYPE_FJW.dat"
def fileName = "/ccb/${curDate}/MODEL_000000_000000_BIGDATA_COMMHOUSETYPE_FJW.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMHOUSETYPE_FJW.dat"
}
@@ -362,7 +362,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMBUILDYEAR_FJW.dat"
def fileName = "/ccb/${curDate}/MODEL_000000_000000_BIGDATA_COMMBUILDYEAR_FJW.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMBUILDYEAR_FJW.dat"
}
@@ -425,7 +425,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMFEE_FJW.dat"
def fileName = "/ccb/${curDate}/MODEL_000000_000000_BIGDATA_COMMFEE_FJW.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMFEE_FJW.dat"
}
@@ -504,7 +504,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMHOUSEUSE_FJW.dat"
def fileName = "/ccb/${curDate}/MODEL_000000_000000_BIGDATA_COMMHOUSEUSE_FJW.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMHOUSEUSE_FJW.dat"
}
@@ -590,7 +590,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
rets << ret
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_POSITION_FJW.dat"
def fileName = "/ccb/${curDate}/MODEL_000000_000000_BIGDATA_POSITION_FJW.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_POSITION_FJW.dat"
}
@@ -5,6 +5,7 @@ import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONObject
@@ -17,7 +18,60 @@ class DevProcess {
static main(args) {
def task = new DevProcess();
task.run()
// task.run()
task.parseCityAndDist()
}
def parseCityAndDist(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\等级开发区.xlsx",{d->
if(!d['J']){
// def obj = getMapInfo(null, d['A']+d['C'])
// if(obj){
// d['E'] = obj.city.toString()
// d['F'] = obj.district.toString()
// d['G'] = obj.formatted_address.toString()
// d['H'] = obj.province.toString()
// }
def obj = FangjiaClientUtils.formatNinelevel(null, d['C']);
if(obj) {
def jsonObject = obj.get("result")
if (jsonObject) {
if (jsonObject.get("province") && jsonObject.get("province") != "null") {
d['J'] = jsonObject.get("province")
}
if (jsonObject.get("city") && jsonObject.get("city") != "null") {
d['K'] = jsonObject.get("city")
}
if (jsonObject.get("district") && jsonObject.get("district") != "null") {
d['L'] = jsonObject.get("district")
}
}
}
}
})
}
def getMapInfo(city, address){
def json = GaoDeApiUtils.geo(city, address)
println json
if(json && json.status=="1"){
def ret = json.geocodes
if(ret){
def obj = ret.get(0)
def str = obj.location
def level = obj.level
// if(str && ["门牌号","单元号","道路","道路交叉路口","热点商圈","村庄","兴趣点","门址","住宅区"].contains(level)){
def arrs = str.split(",")
return ["lng":arrs[0], "lat":arrs[1],"province":obj.province,"city":obj.city, "district":obj.district, "formatted_address":obj.formatted_address]
// }else{
// println address +" "+level
// return null
// }
}
}
}
def run(){
def rets = ExcelUtils.read("D:\\数据\\开发区\\附件全国农村创业园区(基地)目录2021.xls")
@@ -2,11 +2,14 @@ package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.AggregationOutput
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.client.AggregateIterable
import org.bson.Document
/**
* 乡镇街道中间表
@@ -18,12 +21,16 @@ class TownMiddleProcess {
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
DBCollection ADMINISTRATIVE_DIVISION_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division_2023", MONGO100_URL)
DBCollection ADMINISTRATIVE_MAPPING_COLL_100 = MongoUtils.getCollectionByName2("fangjia", "administrative_mapping", MONGO100_URL)
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName("fangjia", "server_area", "mongo.db.fangjia.com")
DBCollection CITYPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "city_polyline")
DBCollection REGIONPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
def fullCityMap = CommonDatautils.queryFullNameCity()
DBCollection TOWNPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline")
def fullCityMap //= CommonDatautils.queryFullNameCity()
static main(args) {
def task = new TownMiddleProcess();
@@ -37,7 +44,13 @@ class TownMiddleProcess {
//创建区域
// createRegion()
//创建乡镇街道
createTown()
// createTown()
//修改乡镇街道别名
// updateTownAlias()
//中间表同步到100
// synchTown()
//添加地级市标志
addCityFlag()
}
@@ -124,7 +137,7 @@ class TownMiddleProcess {
o.pid = provId
}
}
saveToDB(lastRets, "城市")
saveToDB(lastRets, cat)
}
@@ -191,38 +204,101 @@ class TownMiddleProcess {
}
}
//是否为县级市
saveToDB(lastRets, "区域")
saveToDB(lastRets, cat)
}
def createTown(){
def cat = "乡镇街道"
def tjjMaps = findAllProcincesFromTJJ(cat)
def lastRets = []
def govMap = findAllGovTown()
tjjMaps.each {k,v->
def ud = getAdministrativeCodeInfo(k, v, null)
if(govMap && govMap.containsKey(k)){
ud.put("has_gov", govMap.get(k))
}
lastRets << ud
}
//添加区域围栏
def regionPolylineMap = townPolyline()
//添加pid
def provPidMap = createPidData("区域")
//添加区域围栏
def regionPolylineMap = townPolyline()
lastRets.each{o->
def shapeData = regionPolylineMap.get(o.city+"|@|"+o.district)
def shapeData = getStreetPolyline(regionPolylineMap, o, o.district)
if(!shapeData){
def newRegion = o.district
def standardRegion = RegionData.formatRegion(o.city,o.district)
if(standardRegion && standardRegion!=newRegion){
newRegion = standardRegion
shapeData = getStreetPolyline(regionPolylineMap, o, newRegion)
}
}
if(shapeData){
o.shape = shapeData.shape
o.location = shapeData.location
if(shapeData.location){
o.location = shapeData.location
}
}
def provId = provPidMap.get(o.city)
def provId = provPidMap.get(o.city+"@"+o.district)
if(provId){
o.pid = provId
}
}
//是否为县级市
saveToDB(lastRets, "区域")
saveToDB(lastRets, cat)
}
def getStreetPolyline(regionPolylineMap, o, district){
def shapeData = regionPolylineMap.get(o.city+"|@|"+ district+"|@|"+o.street)
if(!shapeData){
def streetKey = o.street
if(streetKey.toString().endsWith("镇")){
streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("镇"))+"街道"
}else if(streetKey.toString().endsWith("乡")){
streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("乡"))+"镇"
}else if(streetKey.toString().endsWith("街道")){
streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("街道"))+"镇"
}else if(streetKey.toString().endsWith("地区")){
streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("地区"))+"镇"
}
shapeData = regionPolylineMap.get(o.city+"|@|"+ district+"|@|"+streetKey)
if(!shapeData){
streetKey = o.street
if(streetKey.toString().endsWith("镇")){
streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("镇"))+"乡"
}else if(streetKey.toString().endsWith("乡")){
streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("乡"))+"镇"
}else if(streetKey.toString().endsWith("街道")){
streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("街道"))+"社区"
}
shapeData = regionPolylineMap.get(o.city+"|@|"+ district+"|@|"+streetKey)
}
if(!shapeData){
streetKey = o.street
if(streetKey.toString().endsWith("镇")){
streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("镇"))+"地区"
}else if(streetKey.toString().endsWith("街道")){
streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("街道"))+"乡"
}
shapeData = regionPolylineMap.get(o.city+"|@|"+ district+"|@|"+streetKey)
}
if(!shapeData){
if(streetKey.toString().endsWith("镇")){
streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("镇"))
}else if(streetKey.toString().endsWith("乡")){
streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("乡"))
}else if(streetKey.toString().endsWith("街道")){
streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("街道"))
}
shapeData = regionPolylineMap.get(o.city+"|@|"+ district+"|@|"+streetKey)
}
}
return shapeData
}
def getAdministrativeCodeInfo(key, val, mzbMaps){
def udata = new BasicDBObject()
def keys = key.split("\\|@\\|")
@@ -269,10 +345,62 @@ class TownMiddleProcess {
udata.put("mzb_district_code", mzb_vals[2])
}
}
if(keys.size()>=4){
udata.put("street", keys[3])
if(vals && vals.size()>=4) {
udata.put("tjj_street_code", vals[3])
}
if(mzb_vals && mzb_vals.size()>=4){
udata.put("mzb_street_code", mzb_vals[3])
}
}
return udata
}
def updateTownAlias(){
ExcelUtils.modify("D:\\数据\\数据建设\\乡镇街道\\乡镇别名问题审核交付_0418.xlsx",{ row->
if(row['G'] && row['G']=="不处理"){
def q = new BasicDBObject()
q.put("city",row['C'])
q.put("district",row['D'])
q.put("street",row['E'])
def d = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
if(d){
def alias = []
alias << row['F']
def udata = new BasicDBObject()
udata.put("alias", alias)
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata))
}else{
println row['C']+" "+row['D']+" "+row["E"]
}
}
})
}
def synchTown(){
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(new BasicDBObject()).batchSize(1000)
def datas = []
for(def d : dbcursor){
datas << d
if (datas.size() >= 2000) {
ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
println datas.size()
datas.clear();
}
}
if (datas.size() >= 0) {
ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
datas.clear();
}
}
def saveToDB(lastRets, type){
def dataList = []
for(def ret : lastRets){
@@ -295,7 +423,7 @@ class TownMiddleProcess {
def createPidData(def type){
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(new BasicDBObject("type", type))
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(new BasicDBObject("type", type)).batchSize(5000)
def map = [:]
def pids = []
def step = 2
@@ -320,7 +448,7 @@ class TownMiddleProcess {
code = d.tjj_province_code.toString()
}else if(type == "城市"){
code = d.tjj_city_code.toString()
}else if(ype == "区域"){
}else if(type == "区域"){
code = d.tjj_district_code.toString()
}
}
@@ -343,6 +471,26 @@ class TownMiddleProcess {
return map;
}
def findAllGovTown(){
def q = new BasicDBObject();
q.put("has_gov", new BasicDBObject('$ne', ''))
def keys = new BasicDBObject()
keys.put("province_name",1)
keys.put("city_name",1)
keys.put("area_name", 1)
keys.put("street_name", 1)
keys.put("has_gov", 1)
def dbcursor = COMMUNITY_CONN.find(q,keys).batchSize(5000)
def map = [:]
for(def d : dbcursor){
def key = d.province_name+"|@|"+d.city_name+"|@|"+d.area_name+"|@|"+d.street_name
if(!map.containsKey(key)){
map.put(key, d.has_gov)
}
}
return map;
}
def findAllProcincesFromTJJ(cat){
// 定义聚合管道,按 city 字段分组,并计算每个城市的数量
@@ -355,14 +503,13 @@ class TownMiddleProcess {
}else if(cat=="区域"){
groupFields.put("_id", new BasicDBObject("province_name", '$province_name').append("province_code", '$province_code').append("city_name", '$city_name').append("city_code", '$city_code').append("area_name", '$area_name').append("area_code", '$area_code'))
}else if(cat=="乡镇街道"){
groupFields.put("_id", new BasicDBObject("province_name", '$province_name').append("province_code", '$province_code').append("city_name", '$city_name').append("city_code", '$city_code').append("area_name", '$area_name').append("area_code", '$area_code').append("name", '$name').append("code", '$code'))
groupFields.put("_id", new BasicDBObject("province_name", '$province_name').append("province_code", '$province_code').append("city_name", '$city_name').append("city_code", '$city_code').append("area_name", '$area_name').append("area_code", '$area_code').append("street_name", '$street_name').append("street_code", '$street_code'))
}
groupFields.put("count", new BasicDBObject('$sum', 1));
DBObject group = new BasicDBObject('$group', groupFields);
// 执行聚合查询
AggregationOutput output = COMMUNITY_CONN.aggregate(Arrays.asList(group));
// AggregationOutput output = COMMUNITY_CONN.aggregate(Arrays.asList(match, group));
def map = [:]
// 遍历结果
@@ -378,8 +525,8 @@ class TownMiddleProcess {
key = key +"|@|"+ obj.get("city_name")+"|@|"+ obj.get("area_name")
value = value +"|@|"+ obj.get("city_code")+"|@|"+ obj.get("area_code")
}else if(cat=="乡镇街道"){
key = key +"|@|"+ obj.get("city_name")+"|@|"+ obj.get("area_name")+"|@|"+ obj.get("name")
value = value +"|@|"+ obj.get("city_code")+"|@|"+ obj.get("area_code")+"|@|"+ obj.get("code")
key = key +"|@|"+ obj.get("city_name")+"|@|"+ obj.get("area_name")+"|@|"+ obj.get("street_name")
value = value +"|@|"+ obj.get("city_code")+"|@|"+ obj.get("area_code")+"|@|"+ obj.get("street_code")
}
int count = (Integer) result.get("count");
map.put(key, value)
@@ -506,7 +653,7 @@ class TownMiddleProcess {
}
def townPolyline(){
def dbcursor = REGIONPOLYLINE_COLL.find(new BasicDBObject("level","district"))
def dbcursor = TOWNPOLYLINE_COLL.find(new BasicDBObject()).batchSize(5000)
def map = [:]
for(def d : dbcursor){
def resultMap = [:]
@@ -526,7 +673,7 @@ class TownMiddleProcess {
if(rets.size()>1) {
resultMap.type = "MultiPolygon"
}
println d.city+" "+d.region+" "+rets.size()
println d.city+" "+d.region+" "+d.town+" "+rets.size()
def udata = new BasicDBObject()
udata.put("shape", resultMap)
@@ -535,10 +682,41 @@ class TownMiddleProcess {
def pointArrs = center.split(",")
udata.put("location", ["lon":pointArrs[0].toString().toDouble(), "lat":pointArrs[1].toString().toDouble()])
}
def fullCity = fullCityMap.get(d.city)
map.put(fullCity.city+"|@|"+d.region, udata)
map.put(d.city+"|@|"+d.fjw_region+"|@|"+d.town, udata)
}
return map;
}
def addCityFlag(){
// ExcelUtils.modify("D:\\数据\\数据建设\\地级市.xlsx",{ row->
// if(row['A']){
// def q = new BasicDBObject("type", "城市")
// q.put("city", row['A'])
// def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
// if(obj){
// def udata = new BasicDBObject()
// udata.put("prefecture_level_city", "是")
// ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata))
// }else{
// println row['A']+" 没找到该城市"
// }
// }
// });
ExcelUtils.modify("D:\\数据\\数据建设\\地级市.xlsx",{ row->
if(row['A']){
def q = new BasicDBObject("type", "区域")
q.put("district", row['A'])
def count = ADMINISTRATIVE_MAPPING_COLL.count(q)
if(count && count == 1){
def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
def udata = new BasicDBObject()
udata.put("county_level_city", "是")
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata))
}else{
println row['A']+" 没找到该城市"
}
}
},1);
}
}
@@ -33,11 +33,11 @@ class AddPriceTest {
task.run();
}
def run(){
def filePath = "D:\\数据\\建行\\房产估值\\估家待优化小区 - 副本.xlsx"
def filePath = "D:\\数据\\建行\\房产估值\\新疆3月待优化小区 - 副本.xlsx"
// addGXDPrice(filePath)
addCcbPrice(filePath)
// addCcbPrice(filePath)
// addThirdPrice(filePath)
// addFjwPrice(filePath)
addFjwPrice(filePath)
// exportPkPrice()
// addRegion()
@@ -160,7 +160,7 @@ class AddPriceTest {
def map = findAvgPrice(d['G'], d['A'].toString().toDouble().toInteger())
if(map){
// d['AK'] = map.avg_price
d['Q'] = map.calc_type
d['P'] = map.calc_type
// d['AM'] = map.credit_value
}
}
@@ -262,7 +262,7 @@ class AddPriceTest {
// d['N'] = dist.region
def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name)
if(dbo){
d['P'] = dbo.avg_price
d['O'] = dbo.avg_price
// d['J'] = dbo.avg_price_2b
}
}