Merge branch 'script_gj' into 'master'

update

See merge request !893
This commit is contained in:
2024-08-08 15:57:24 +08:00
4 changed files with 71 additions and 13 deletions
@@ -40,7 +40,7 @@ class BigdataCommunityBatch extends BaseBatch{
@Override @Override
public void process(Object args) { public void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args) def citys = CcbCommonUtils.getRunningCitys(args)
def rows = [] def rows = []
def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途", def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途",
@@ -50,6 +50,7 @@ class BigdataCommunityBatch extends BaseBatch{
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def index = 0 def index = 0
def today = new Date().getDate() def today = new Date().getDate()
def fangjiaDists = []
println "starting 。。。。。。。。。" println "starting 。。。。。。。。。"
for(def city : citys){ for(def city : citys){
@@ -60,6 +61,9 @@ class BigdataCommunityBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"], null) def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"], null)
for(def d : dists){
fangjiaDists << d._id.toString()
}
def ccbMaps = null def ccbMaps = null
if(today<=10){ if(today<=10){
@@ -69,7 +73,7 @@ class BigdataCommunityBatch extends BaseBatch{
} }
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists) def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
rets = addDeleteDist(dists, ccbMaps, rets) rets = addDeleteDist(dists, ccbMaps, rets, fangjiaDists)
rows.addAll(rets) rows.addAll(rets)
println "[${index}] "+city +" 小区量:"+dists.size()+" 总:"+rets.size() println "[${index}] "+city +" 小区量:"+dists.size()+" 总:"+rets.size()
@@ -102,11 +106,8 @@ class BigdataCommunityBatch extends BaseBatch{
} }
def addDeleteDist(dists, ccbMaps, rets){ def addDeleteDist(dists, ccbMaps, rets, fangjiaDists){
def fangjiaDists = []
for(def d : dists){
fangjiaDists << d._id.toString()
}
println "房价网小区:"+fangjiaDists.size()+" 建行小区:"+rets.size() println "房价网小区:"+fangjiaDists.size()+" 建行小区:"+rets.size()
def ccbDists = ccbMaps.keySet() def ccbDists = ccbMaps.keySet()
for(def ccbId : ccbDists){ for(def ccbId : ccbDists){
@@ -4653,6 +4653,7 @@ class CommonDatautils {
def q = new BasicDBObject() def q = new BasicDBObject()
q.put("type","区域") q.put("type","区域")
q.put("district_diff",1) q.put("district_diff",1)
q.put("city",new BasicDBObject('$in', ["中山","东莞","东方"]))
def keys = new BasicDBObject() def keys = new BasicDBObject()
keys.put("province",1) keys.put("province",1)
@@ -4662,7 +4663,7 @@ class CommonDatautils {
keys.put("mzb_district_name", 1) keys.put("mzb_district_name", 1)
keys.put("shape", 1) keys.put("shape", 1)
keys.put("district_short",1) keys.put("district_short",1)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000) def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def regionMap = [:] def regionMap = [:]
for(def d : dbcursor){ for(def d : dbcursor){
if(d.mzb_district_name){ if(d.mzb_district_name){
@@ -4686,7 +4687,7 @@ class CommonDatautils {
def q = new BasicDBObject() def q = new BasicDBObject()
q.put("type","乡镇街道") q.put("type","乡镇街道")
// q.put("province",new BasicDBObject('$in', ['浙江省','江苏省','山东省'])) // q.put("province",new BasicDBObject('$in', ['浙江省','江苏省','山东省']))
// q.put("city","芜湖市") q.put("city",new BasicDBObject('$in', ["中山市","东莞市","东方市"]))
def keys = new BasicDBObject() def keys = new BasicDBObject()
keys.put("province",1) keys.put("province",1)
@@ -4712,10 +4713,42 @@ class CommonDatautils {
} }
static def queryTownByCity(city){
def q = new BasicDBObject()
q.put("type","乡镇街道")
q.put("city",city)
def keys = new BasicDBObject()
keys.put("province",1)
keys.put("city",1)
keys.put("district",1)
keys.put("street",1)
keys.put("shape",1)
keys.put("alias",1)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def townMap = [:]
for(def d : dbcursor){
def key = d.district
def list = townMap.get(key)
if(!list){
list = []
}
list << d
townMap.put(key, list)
}
println "乡镇街道 加载完成"
return townMap
}
static def queryNewTownByCity(){ static def queryNewTownByCity(){
def q = new BasicDBObject() def q = new BasicDBObject()
q.put("type","乡镇街道") q.put("type","乡镇街道")
q.put("city",new BasicDBObject('$in', ["中山","东莞","东方"]))
def keys = new BasicDBObject() def keys = new BasicDBObject()
keys.put("city",1) keys.put("city",1)
@@ -4833,6 +4866,9 @@ class CommonDatautils {
def jsonObject = null def jsonObject = null
def stop = false def stop = false
def count = 0 def count = 0
def road = ""
def name = ""
def type = ""
while(!stop){ while(!stop){
if(glocation){ if(glocation){
jsonObject = GaoDeApiUtils.geocode(glocation) jsonObject = GaoDeApiUtils.geocode(glocation)
@@ -4852,8 +4888,26 @@ class CommonDatautils {
} }
format_addr = jsonObject.regeocode.formatted_address
if(addressComponent.streetNumber){
if(addressComponent.streetNumber.street){
road = addressComponent.streetNumber.street
}
if(addressComponent.streetNumber.number){
road = road + addressComponent.streetNumber.number
}
}
if(addressComponent.building){
if(addressComponent.building.name){
name = addressComponent.building.name
}
if(addressComponent.building.type){
type = addressComponent.building.type
}
}
format_addr = jsonObject.regeocode.formatted_address
} }
} }
} }
@@ -4871,6 +4925,9 @@ class CommonDatautils {
map.district = district map.district = district
map.township = township map.township = township
map.format_addr = format_addr map.format_addr = format_addr
map.road = road
map.type = type
map.name = name
return map; return map;
} }
@@ -520,8 +520,8 @@ class FjwBigdataDealBatch extends BaseBatch{
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) // def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date()) // def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20200101", "yyyyMMdd")) def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20240501", "yyyyMMdd"))
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20240601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date()) def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20240701", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
for(def i=0;i<dists.size();i++){ for(def i=0;i<dists.size();i++){
@@ -53,7 +53,7 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
def stop = false def stop = false
def month = -1 def month = -1
while(!stop){ while(!stop){
if(sDate<202004){ if(sDate>202404 && sDate<202407){
def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd")) def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd"))
println weekend.toString() +" "+sDate println weekend.toString() +" "+sDate
runDatas(citys) runDatas(citys)