Merge branch 'script_gj' into 'master'

update

See merge request !893
This commit is contained in:
2024-08-08 15:57:24 +08:00
4 changed files with 71 additions and 13 deletions
@@ -40,7 +40,7 @@ class BigdataCommunityBatch extends BaseBatch{
@Override
public void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args)
def citys = CcbCommonUtils.getRunningCitys(args)
def rows = []
def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途",
@@ -50,6 +50,7 @@ class BigdataCommunityBatch extends BaseBatch{
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def index = 0
def today = new Date().getDate()
def fangjiaDists = []
println "starting 。。。。。。。。。"
for(def city : citys){
@@ -60,6 +61,9 @@ class BigdataCommunityBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"], null)
for(def d : dists){
fangjiaDists << d._id.toString()
}
def ccbMaps = null
if(today<=10){
@@ -69,7 +73,7 @@ class BigdataCommunityBatch extends BaseBatch{
}
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
rets = addDeleteDist(dists, ccbMaps, rets)
rets = addDeleteDist(dists, ccbMaps, rets, fangjiaDists)
rows.addAll(rets)
println "[${index}] "+city +" 小区量:"+dists.size()+" 总:"+rets.size()
@@ -102,11 +106,8 @@ class BigdataCommunityBatch extends BaseBatch{
}
def addDeleteDist(dists, ccbMaps, rets){
def fangjiaDists = []
for(def d : dists){
fangjiaDists << d._id.toString()
}
def addDeleteDist(dists, ccbMaps, rets, fangjiaDists){
println "房价网小区:"+fangjiaDists.size()+" 建行小区:"+rets.size()
def ccbDists = ccbMaps.keySet()
for(def ccbId : ccbDists){
@@ -4653,6 +4653,7 @@ class CommonDatautils {
def q = new BasicDBObject()
q.put("type","区域")
q.put("district_diff",1)
q.put("city",new BasicDBObject('$in', ["中山","东莞","东方"]))
def keys = new BasicDBObject()
keys.put("province",1)
@@ -4662,7 +4663,7 @@ class CommonDatautils {
keys.put("mzb_district_name", 1)
keys.put("shape", 1)
keys.put("district_short",1)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def regionMap = [:]
for(def d : dbcursor){
if(d.mzb_district_name){
@@ -4686,7 +4687,7 @@ class CommonDatautils {
def q = new BasicDBObject()
q.put("type","乡镇街道")
// q.put("province",new BasicDBObject('$in', ['浙江省','江苏省','山东省']))
// q.put("city","芜湖市")
q.put("city",new BasicDBObject('$in', ["中山市","东莞市","东方市"]))
def keys = new BasicDBObject()
keys.put("province",1)
@@ -4712,10 +4713,42 @@ class CommonDatautils {
}
static def queryTownByCity(city){
def q = new BasicDBObject()
q.put("type","乡镇街道")
q.put("city",city)
def keys = new BasicDBObject()
keys.put("province",1)
keys.put("city",1)
keys.put("district",1)
keys.put("street",1)
keys.put("shape",1)
keys.put("alias",1)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def townMap = [:]
for(def d : dbcursor){
def key = d.district
def list = townMap.get(key)
if(!list){
list = []
}
list << d
townMap.put(key, list)
}
println "乡镇街道 加载完成"
return townMap
}
static def queryNewTownByCity(){
def q = new BasicDBObject()
q.put("type","乡镇街道")
q.put("city",new BasicDBObject('$in', ["中山","东莞","东方"]))
def keys = new BasicDBObject()
keys.put("city",1)
@@ -4833,6 +4866,9 @@ class CommonDatautils {
def jsonObject = null
def stop = false
def count = 0
def road = ""
def name = ""
def type = ""
while(!stop){
if(glocation){
jsonObject = GaoDeApiUtils.geocode(glocation)
@@ -4852,8 +4888,26 @@ class CommonDatautils {
}
format_addr = jsonObject.regeocode.formatted_address
if(addressComponent.streetNumber){
if(addressComponent.streetNumber.street){
road = addressComponent.streetNumber.street
}
if(addressComponent.streetNumber.number){
road = road + addressComponent.streetNumber.number
}
}
if(addressComponent.building){
if(addressComponent.building.name){
name = addressComponent.building.name
}
if(addressComponent.building.type){
type = addressComponent.building.type
}
}
format_addr = jsonObject.regeocode.formatted_address
}
}
}
@@ -4871,6 +4925,9 @@ class CommonDatautils {
map.district = district
map.township = township
map.format_addr = format_addr
map.road = road
map.type = type
map.name = name
return map;
}
@@ -520,8 +520,8 @@ class FjwBigdataDealBatch extends BaseBatch{
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20200101", "yyyyMMdd"))
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20240601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20240501", "yyyyMMdd"))
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20240701", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
for(def i=0;i<dists.size();i++){
@@ -53,7 +53,7 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
def stop = false
def month = -1
while(!stop){
if(sDate<202004){
if(sDate>202404 && sDate<202407){
def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd"))
println weekend.toString() +" "+sDate
runDatas(citys)