update
This commit is contained in:
+8
-7
@@ -40,7 +40,7 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def rows = []
|
||||
|
||||
def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途",
|
||||
@@ -50,6 +50,7 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def index = 0
|
||||
def today = new Date().getDate()
|
||||
def fangjiaDists = []
|
||||
|
||||
println "starting 。。。。。。。。。"
|
||||
for(def city : citys){
|
||||
@@ -60,6 +61,9 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"], null)
|
||||
|
||||
for(def d : dists){
|
||||
fangjiaDists << d._id.toString()
|
||||
}
|
||||
|
||||
def ccbMaps = null
|
||||
if(today<=10){
|
||||
@@ -69,7 +73,7 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
|
||||
rets = addDeleteDist(dists, ccbMaps, rets)
|
||||
rets = addDeleteDist(dists, ccbMaps, rets, fangjiaDists)
|
||||
rows.addAll(rets)
|
||||
println "[${index}] "+city +" 小区量:"+dists.size()+" 总:"+rets.size()
|
||||
|
||||
@@ -102,11 +106,8 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
|
||||
}
|
||||
|
||||
def addDeleteDist(dists, ccbMaps, rets){
|
||||
def fangjiaDists = []
|
||||
for(def d : dists){
|
||||
fangjiaDists << d._id.toString()
|
||||
}
|
||||
def addDeleteDist(dists, ccbMaps, rets, fangjiaDists){
|
||||
|
||||
println "房价网小区:"+fangjiaDists.size()+" 建行小区:"+rets.size()
|
||||
def ccbDists = ccbMaps.keySet()
|
||||
for(def ccbId : ccbDists){
|
||||
|
||||
+60
-3
@@ -4653,6 +4653,7 @@ class CommonDatautils {
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","区域")
|
||||
q.put("district_diff",1)
|
||||
q.put("city",new BasicDBObject('$in', ["中山","东莞","东方"]))
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("province",1)
|
||||
@@ -4662,7 +4663,7 @@ class CommonDatautils {
|
||||
keys.put("mzb_district_name", 1)
|
||||
keys.put("shape", 1)
|
||||
keys.put("district_short",1)
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000)
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def regionMap = [:]
|
||||
for(def d : dbcursor){
|
||||
if(d.mzb_district_name){
|
||||
@@ -4686,7 +4687,7 @@ class CommonDatautils {
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","乡镇街道")
|
||||
// q.put("province",new BasicDBObject('$in', ['浙江省','江苏省','山东省']))
|
||||
// q.put("city","芜湖市")
|
||||
q.put("city",new BasicDBObject('$in', ["中山市","东莞市","东方市"]))
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("province",1)
|
||||
@@ -4712,10 +4713,42 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
|
||||
|
||||
static def queryTownByCity(city){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","乡镇街道")
|
||||
q.put("city",city)
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("province",1)
|
||||
keys.put("city",1)
|
||||
keys.put("district",1)
|
||||
keys.put("street",1)
|
||||
keys.put("shape",1)
|
||||
keys.put("alias",1)
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def townMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def key = d.district
|
||||
def list = townMap.get(key)
|
||||
if(!list){
|
||||
list = []
|
||||
}
|
||||
list << d
|
||||
townMap.put(key, list)
|
||||
}
|
||||
|
||||
println "乡镇街道 加载完成"
|
||||
return townMap
|
||||
}
|
||||
|
||||
|
||||
static def queryNewTownByCity(){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","乡镇街道")
|
||||
q.put("city",new BasicDBObject('$in', ["中山","东莞","东方"]))
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("city",1)
|
||||
@@ -4833,6 +4866,9 @@ class CommonDatautils {
|
||||
def jsonObject = null
|
||||
def stop = false
|
||||
def count = 0
|
||||
def road = ""
|
||||
def name = ""
|
||||
def type = ""
|
||||
while(!stop){
|
||||
if(glocation){
|
||||
jsonObject = GaoDeApiUtils.geocode(glocation)
|
||||
@@ -4852,8 +4888,26 @@ class CommonDatautils {
|
||||
|
||||
}
|
||||
|
||||
format_addr = jsonObject.regeocode.formatted_address
|
||||
|
||||
if(addressComponent.streetNumber){
|
||||
if(addressComponent.streetNumber.street){
|
||||
road = addressComponent.streetNumber.street
|
||||
}
|
||||
if(addressComponent.streetNumber.number){
|
||||
road = road + addressComponent.streetNumber.number
|
||||
}
|
||||
}
|
||||
|
||||
if(addressComponent.building){
|
||||
if(addressComponent.building.name){
|
||||
name = addressComponent.building.name
|
||||
}
|
||||
if(addressComponent.building.type){
|
||||
type = addressComponent.building.type
|
||||
}
|
||||
}
|
||||
|
||||
format_addr = jsonObject.regeocode.formatted_address
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4871,6 +4925,9 @@ class CommonDatautils {
|
||||
map.district = district
|
||||
map.township = township
|
||||
map.format_addr = format_addr
|
||||
map.road = road
|
||||
map.type = type
|
||||
map.name = name
|
||||
return map;
|
||||
}
|
||||
|
||||
|
||||
+2
-2
@@ -520,8 +520,8 @@ class FjwBigdataDealBatch extends BaseBatch{
|
||||
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
|
||||
|
||||
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20200101", "yyyyMMdd"))
|
||||
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20240601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
|
||||
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20240501", "yyyyMMdd"))
|
||||
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20240701", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
|
||||
|
||||
|
||||
for(def i=0;i<dists.size();i++){
|
||||
|
||||
+1
-1
@@ -53,7 +53,7 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
|
||||
def stop = false
|
||||
def month = -1
|
||||
while(!stop){
|
||||
if(sDate<202004){
|
||||
if(sDate>202404 && sDate<202407){
|
||||
def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd"))
|
||||
println weekend.toString() +" "+sDate
|
||||
runDatas(citys)
|
||||
|
||||
Reference in New Issue
Block a user