feat(office): auto-calculate mom and yoy for city, district, and office monthly stats
This commit is contained in:
@@ -186,6 +186,49 @@ def calculate_mom_yoy_for_office(city, district, office_name):
|
||||
office_rent_stat_conn.bulk_write(actions)
|
||||
|
||||
|
||||
def calculate_mom_yoy_by_month(year_month, city=None):
|
||||
"""
|
||||
批量计算某月所有写字楼的 mom (环比) 和 yoy (同比) 并更新
|
||||
"""
|
||||
curr_date = int(year_month)
|
||||
year = curr_date // 100
|
||||
month = curr_date % 100
|
||||
prev_date = (year - 1) * 100 + 12 if month == 1 else curr_date - 1
|
||||
ly_date = curr_date - 100
|
||||
|
||||
base_query = {'cat': 'office'}
|
||||
if pd.notna(city) and city != '':
|
||||
base_query['city'] = city
|
||||
|
||||
proj = {'_id': 1, 'city': 1, 'district': 1, 'name': 1, 'avg_price': 1}
|
||||
curr_docs = list(office_rent_stat_conn.find({**base_query, 'date': curr_date}, proj))
|
||||
if not curr_docs:
|
||||
return
|
||||
|
||||
prev_docs = list(office_rent_stat_conn.find({**base_query, 'date': prev_date}, proj))
|
||||
ly_docs = list(office_rent_stat_conn.find({**base_query, 'date': ly_date}, proj))
|
||||
|
||||
prev_map = {(d.get('city'), d.get('district'), d.get('name')): d.get('avg_price') for d in prev_docs if d.get('avg_price')}
|
||||
ly_map = {(d.get('city'), d.get('district'), d.get('name')): d.get('avg_price') for d in ly_docs if d.get('avg_price')}
|
||||
|
||||
actions = []
|
||||
for doc in curr_docs:
|
||||
curr_price = doc.get('avg_price')
|
||||
if not curr_price:
|
||||
continue
|
||||
key = (doc.get('city'), doc.get('district'), doc.get('name'))
|
||||
prev_price = prev_map.get(key)
|
||||
ly_price = ly_map.get(key)
|
||||
mom = round((curr_price - prev_price) / prev_price * 100, 2) if prev_price else None
|
||||
yoy = round((curr_price - ly_price) / ly_price * 100, 2) if ly_price else None
|
||||
actions.append(UpdateOne({'_id': doc['_id']}, {'$set': {'mom': mom, 'yoy': yoy}}))
|
||||
if len(actions) == 5000:
|
||||
office_rent_stat_conn.bulk_write(actions)
|
||||
actions.clear()
|
||||
if actions:
|
||||
office_rent_stat_conn.bulk_write(actions)
|
||||
|
||||
|
||||
def calcByCity(city, year_month):
|
||||
"""
|
||||
写字楼基准价
|
||||
@@ -212,6 +255,7 @@ def calcByCity(city, year_month):
|
||||
office = getOffice(city)
|
||||
|
||||
update_office(city, office, sample, year_month)
|
||||
calculate_mom_yoy_by_month(year_month, city)
|
||||
|
||||
|
||||
def update_office(city, office, sample, year_month):
|
||||
|
||||
+36
-16
@@ -37,25 +37,36 @@ def calc(city, year_month):
|
||||
region_median_count_prices = {'median': {}, 'count': {}}
|
||||
|
||||
actions = list()
|
||||
|
||||
year_month = int(year_month)
|
||||
last_month = int(
|
||||
dateutils.format_date(dateutils.add_months(dateutils.parse_date(str(year_month), '%Y%m'), months=-1), '%Y%m'))
|
||||
ly_month = year_month - 100
|
||||
|
||||
prev_city_stat = office_rent_stat_conn.find_one(
|
||||
{'city': city, 'cat': 'city', 'date': {'$lte': int(last_month)}, 'avg_price': {'$lte': 15, '$gt': 0}},
|
||||
sort=[('date', -1)]
|
||||
)
|
||||
if pd.isna(city_price):
|
||||
# 处理均价为空情况,用上月均价替代
|
||||
stat = office_rent_stat_conn.find_one(
|
||||
{'city': city, 'cat': 'city', 'date': {'$lte': int(last_month)}, 'avg_price': {'$lte': 15, '$gt': 0}},
|
||||
sort=[('date', -1)]
|
||||
)
|
||||
if pd.isna(stat):
|
||||
if pd.isna(prev_city_stat):
|
||||
return False
|
||||
last_month_city_price = stat['avg_price']
|
||||
last_month_city_price = prev_city_stat['avg_price']
|
||||
if pd.isna(last_month_city_price):
|
||||
return False
|
||||
city_price = last_month_city_price
|
||||
|
||||
ly_city_stat = office_rent_stat_conn.find_one(
|
||||
{'city': city, 'cat': 'city', 'date': int(ly_month), 'avg_price': {'$lte': 15, '$gt': 0}}
|
||||
)
|
||||
city_mom = round((city_price - prev_city_stat['avg_price']) / prev_city_stat['avg_price'] * 100, 2) if (
|
||||
prev_city_stat and prev_city_stat.get('avg_price')) else None
|
||||
city_yoy = round((city_price - ly_city_stat['avg_price']) / ly_city_stat['avg_price'] * 100, 2) if (
|
||||
ly_city_stat and ly_city_stat.get('avg_price')) else None
|
||||
|
||||
# 写入MONGODB
|
||||
action = UpdateOne({'city': city, 'cat': 'city', 'date': int(year_month)},
|
||||
{'$set': {'city': city, 'cat': 'city', 'date': int(year_month), 'avg_price': city_price,
|
||||
'mom': city_mom, 'yoy': city_yoy,
|
||||
'updated_date': datetime.now()}},
|
||||
upsert=True)
|
||||
|
||||
@@ -63,24 +74,33 @@ def calc(city, year_month):
|
||||
for region in areautils.get_all_districts(city):
|
||||
region_avg_prices = region_median_count_prices['median']
|
||||
region_count = region_median_count_prices['count']
|
||||
prev_district_stat = office_rent_stat_conn.find_one(
|
||||
{'city': city, 'cat': 'district', 'date': {'$lte': int(last_month)}, 'district': region, 'avg_price': {'$lte': 15, '$gt': 0}},
|
||||
sort=[('date', -1)]
|
||||
)
|
||||
if region in region_avg_prices and region_count[region] >= 10:
|
||||
avg_price = region_avg_prices[region]
|
||||
avg_price = round(region_avg_prices[region], 2)
|
||||
else:
|
||||
# 处理均价为空情况,用上月均价替代
|
||||
district_stat = office_rent_stat_conn.find_one(
|
||||
{'city': city, 'cat': 'district', 'date': {'$lte': int(last_month)}, 'district': region, 'avg_price': {'$lte': 15, '$gt': 0}},
|
||||
sort=[('date', -1)]
|
||||
)
|
||||
if pd.isna(district_stat):
|
||||
if pd.isna(prev_district_stat):
|
||||
continue
|
||||
last_month_region_price = district_stat['avg_price']
|
||||
last_month_region_price = prev_district_stat['avg_price']
|
||||
if pd.isna(last_month_region_price):
|
||||
continue
|
||||
avg_price = last_month_region_price
|
||||
avg_price = round(last_month_region_price, 2)
|
||||
|
||||
ly_district_stat = office_rent_stat_conn.find_one(
|
||||
{'city': city, 'cat': 'district', 'date': int(ly_month), 'district': region, 'avg_price': {'$lte': 15, '$gt': 0}}
|
||||
)
|
||||
region_mom = round((avg_price - prev_district_stat['avg_price']) / prev_district_stat['avg_price'] * 100, 2) if (
|
||||
prev_district_stat and prev_district_stat.get('avg_price')) else None
|
||||
region_yoy = round((avg_price - ly_district_stat['avg_price']) / ly_district_stat['avg_price'] * 100, 2) if (
|
||||
ly_district_stat and ly_district_stat.get('avg_price')) else None
|
||||
|
||||
action = UpdateOne({'city': city, 'cat': 'district', 'date': int(year_month), 'district': region},
|
||||
{'$set': {'city': city, 'cat': 'district', 'date': int(year_month), 'district': region,
|
||||
'avg_price': round(avg_price, 2), 'updated_date': datetime.now()}},
|
||||
'avg_price': avg_price, 'mom': region_mom, 'yoy': region_yoy,
|
||||
'updated_date': datetime.now()}},
|
||||
upsert=True)
|
||||
actions.append(action)
|
||||
office_rent_stat_conn.bulk_write(actions)
|
||||
|
||||
@@ -187,6 +187,49 @@ def calculate_mom_yoy_for_office(city, district, office_name):
|
||||
office_sell_stat_conn.bulk_write(actions)
|
||||
|
||||
|
||||
def calculate_mom_yoy_by_month(year_month, city=None):
|
||||
"""
|
||||
批量计算某月所有写字楼出售的 mom (环比) 和 yoy (同比) 并更新
|
||||
"""
|
||||
curr_date = int(year_month)
|
||||
year = curr_date // 100
|
||||
month = curr_date % 100
|
||||
prev_date = (year - 1) * 100 + 12 if month == 1 else curr_date - 1
|
||||
ly_date = curr_date - 100
|
||||
|
||||
base_query = {'cat': 'office'}
|
||||
if pd.notna(city) and city != '':
|
||||
base_query['city'] = city
|
||||
|
||||
proj = {'_id': 1, 'city': 1, 'district': 1, 'name': 1, 'avg_price': 1}
|
||||
curr_docs = list(office_sell_stat_conn.find({**base_query, 'date': curr_date}, proj))
|
||||
if not curr_docs:
|
||||
return
|
||||
|
||||
prev_docs = list(office_sell_stat_conn.find({**base_query, 'date': prev_date}, proj))
|
||||
ly_docs = list(office_sell_stat_conn.find({**base_query, 'date': ly_date}, proj))
|
||||
|
||||
prev_map = {(d.get('city'), d.get('district'), d.get('name')): d.get('avg_price') for d in prev_docs if d.get('avg_price')}
|
||||
ly_map = {(d.get('city'), d.get('district'), d.get('name')): d.get('avg_price') for d in ly_docs if d.get('avg_price')}
|
||||
|
||||
actions = []
|
||||
for doc in curr_docs:
|
||||
curr_price = doc.get('avg_price')
|
||||
if not curr_price:
|
||||
continue
|
||||
key = (doc.get('city'), doc.get('district'), doc.get('name'))
|
||||
prev_price = prev_map.get(key)
|
||||
ly_price = ly_map.get(key)
|
||||
mom = round((curr_price - prev_price) / prev_price * 100, 2) if prev_price else None
|
||||
yoy = round((curr_price - ly_price) / ly_price * 100, 2) if ly_price else None
|
||||
actions.append(UpdateOne({'_id': doc['_id']}, {'$set': {'mom': mom, 'yoy': yoy}}))
|
||||
if len(actions) == 5000:
|
||||
office_sell_stat_conn.bulk_write(actions)
|
||||
actions.clear()
|
||||
if actions:
|
||||
office_sell_stat_conn.bulk_write(actions)
|
||||
|
||||
|
||||
def calcByCity(city, year_month):
|
||||
"""
|
||||
写字楼基准价
|
||||
@@ -214,6 +257,7 @@ def calcByCity(city, year_month):
|
||||
office = getOffice(city)
|
||||
|
||||
update_office(city, office, sample, year_month)
|
||||
calculate_mom_yoy_by_month(year_month, city)
|
||||
|
||||
|
||||
def update_office(city, office, sample, year_month):
|
||||
|
||||
@@ -82,20 +82,34 @@ def calc_by_yearmonth(office_sell, office_rent, year_month, office_sell_stat_con
|
||||
merge2_df = threads.process_dataframe_multithread(merge2_df, deal, 10)
|
||||
merge2_df = merge2_df[merge2_df['sell_price'].notna()]
|
||||
|
||||
curr_date = int(year_month)
|
||||
year = curr_date // 100
|
||||
month = curr_date % 100
|
||||
prev_date = (year - 1) * 100 + 12 if month == 1 else curr_date - 1
|
||||
ly_date = curr_date - 100
|
||||
proj = {'_id': 0, 'city': 1, 'district': 1, 'name': 1, 'avg_price': 1}
|
||||
prev_map = {(d.get('city'), d.get('district'), d.get('name')): d.get('avg_price') for d in office_sell_stat_conn.find({'cat': 'office', 'date': prev_date}, proj) if d.get('avg_price')}
|
||||
ly_map = {(d.get('city'), d.get('district'), d.get('name')): d.get('avg_price') for d in office_sell_stat_conn.find({'cat': 'office', 'date': ly_date}, proj) if d.get('avg_price')}
|
||||
|
||||
actions = list()
|
||||
count = 0
|
||||
for index, row in merge2_df.iterrows():
|
||||
curr_city = row['city']
|
||||
region = row['district']
|
||||
name = row['name']
|
||||
sell_price = row['sell_price']
|
||||
sell_price = int(row['sell_price'])
|
||||
estate_type2 = row['estate_type2']
|
||||
bed_kind = row['bed_kind']
|
||||
location = row['location']
|
||||
action = UpdateOne({'cat': 'office', 'city': curr_city, 'district': region, 'name': name, 'date': year_month},
|
||||
{'$set': {'city': curr_city, 'cat': 'office', 'date': year_month, 'district': region,
|
||||
prev_p = prev_map.get((curr_city, region, name))
|
||||
ly_p = ly_map.get((curr_city, region, name))
|
||||
mom = round((sell_price - prev_p) / prev_p * 100, 2) if prev_p else None
|
||||
yoy = round((sell_price - ly_p) / ly_p * 100, 2) if ly_p else None
|
||||
action = UpdateOne({'cat': 'office', 'city': curr_city, 'district': region, 'name': name, 'date': curr_date},
|
||||
{'$set': {'city': curr_city, 'cat': 'office', 'date': curr_date, 'district': region,
|
||||
'name': name,
|
||||
'avg_price': int(sell_price), 'estate_type2': estate_type2, 'bed_kind': bed_kind,
|
||||
'avg_price': sell_price, 'mom': mom, 'yoy': yoy,
|
||||
'estate_type2': estate_type2, 'bed_kind': bed_kind,
|
||||
'location': location,
|
||||
'calc_type': 5, 'updated_date': datetime.now()}},
|
||||
upsert=True)
|
||||
|
||||
@@ -59,11 +59,25 @@ def calc(city, year_month):
|
||||
merge2_df = threads.process_dataframe_multithread(merge2_df, deal_city, 10)
|
||||
merge2_df = merge2_df[merge2_df['sell_price'].notna()]
|
||||
|
||||
year = year_month // 100
|
||||
month = year_month % 100
|
||||
prev_date = (year - 1) * 100 + 12 if month == 1 else year_month - 1
|
||||
ly_date = year_month - 100
|
||||
|
||||
prev_city_map = {d.get('city'): d.get('avg_price') for d in office_sell_stat_conn.find({'date': prev_date, 'cat': 'city'}) if d.get('avg_price')}
|
||||
ly_city_map = {d.get('city'): d.get('avg_price') for d in office_sell_stat_conn.find({'date': ly_date, 'cat': 'city'}) if d.get('avg_price')}
|
||||
|
||||
actions = list()
|
||||
# 写入MONGODB
|
||||
for index, row in merge2_df.iterrows():
|
||||
sell_p = int(row['sell_price'])
|
||||
prev_p = prev_city_map.get(row['city'])
|
||||
ly_p = ly_city_map.get(row['city'])
|
||||
mom = round((sell_p - prev_p) / prev_p * 100, 2) if prev_p else None
|
||||
yoy = round((sell_p - ly_p) / ly_p * 100, 2) if ly_p else None
|
||||
action = UpdateOne({'city': row['city'], 'cat': 'city', 'date': year_month},
|
||||
{'$set': {'city': row['city'], 'cat': 'city', 'date': year_month, 'calc_type': 2, 'avg_price': int(row['sell_price']),
|
||||
{'$set': {'city': row['city'], 'cat': 'city', 'date': year_month, 'calc_type': 2, 'avg_price': sell_p,
|
||||
'mom': mom, 'yoy': yoy,
|
||||
'updated_date': datetime.now()}},
|
||||
upsert=True)
|
||||
actions.append(action)
|
||||
@@ -89,10 +103,19 @@ def calc(city, year_month):
|
||||
merge2_df = threads.process_dataframe_multithread(merge2_df, deal_district, 10)
|
||||
merge2_df = merge2_df[merge2_df['sell_price'].notna()]
|
||||
|
||||
prev_dist_map = {(d.get('city'), d.get('district')): d.get('avg_price') for d in office_sell_stat_conn.find({'date': prev_date, 'cat': 'district'}) if d.get('avg_price')}
|
||||
ly_dist_map = {(d.get('city'), d.get('district')): d.get('avg_price') for d in office_sell_stat_conn.find({'date': ly_date, 'cat': 'district'}) if d.get('avg_price')}
|
||||
|
||||
# 写入MONGODB
|
||||
for index, row in merge2_df.iterrows():
|
||||
sell_p = int(row['sell_price'])
|
||||
prev_p = prev_dist_map.get((row['city'], row['district']))
|
||||
ly_p = ly_dist_map.get((row['city'], row['district']))
|
||||
mom = round((sell_p - prev_p) / prev_p * 100, 2) if prev_p else None
|
||||
yoy = round((sell_p - ly_p) / ly_p * 100, 2) if ly_p else None
|
||||
action = UpdateOne({'city': row['city'], 'district': row['district'], 'cat': 'district', 'date': year_month},
|
||||
{'$set': {'city': row['city'], 'district': row['district'], 'cat': 'district', 'date': year_month, 'calc_type': 2, 'avg_price': int(row['sell_price']),
|
||||
{'$set': {'city': row['city'], 'district': row['district'], 'cat': 'district', 'date': year_month, 'calc_type': 2, 'avg_price': sell_p,
|
||||
'mom': mom, 'yoy': yoy,
|
||||
'updated_date': datetime.now()}},
|
||||
upsert=True)
|
||||
actions.append(action)
|
||||
|
||||
+39
-11
@@ -37,22 +37,38 @@ def calc(city, year_month):
|
||||
region_median_count_prices = {'median': {}, 'count': {}}
|
||||
|
||||
actions = list()
|
||||
|
||||
year_month = int(year_month)
|
||||
last_month = int(
|
||||
dateutils.format_date(dateutils.add_months(dateutils.parse_date(str(year_month), '%Y%m'), months=-1), '%Y%m'))
|
||||
ly_month = year_month - 100
|
||||
|
||||
prev_city_stat = office_sell_stat_conn.find_one(
|
||||
{'city': city, 'cat': 'city', 'date': {'$lte': last_month}, 'avg_price': {'$gt': 0}},
|
||||
sort=[('date', -1)]
|
||||
)
|
||||
if pd.isna(city_price):
|
||||
# 处理均价为空情况,用上月均价替代
|
||||
stat = office_sell_stat_conn.find_one({'city': city, 'cat': 'city', 'date': last_month})
|
||||
if pd.isna(stat):
|
||||
if pd.isna(prev_city_stat):
|
||||
return False
|
||||
last_month_city_price = stat['avg_price']
|
||||
last_month_city_price = prev_city_stat['avg_price']
|
||||
if pd.isna(last_month_city_price):
|
||||
return False
|
||||
city_price = last_month_city_price
|
||||
city_price = int(last_month_city_price)
|
||||
else:
|
||||
city_price = int(city_price)
|
||||
|
||||
ly_city_stat = office_sell_stat_conn.find_one(
|
||||
{'city': city, 'cat': 'city', 'date': ly_month, 'avg_price': {'$gt': 0}}
|
||||
)
|
||||
city_mom = round((city_price - prev_city_stat['avg_price']) / prev_city_stat['avg_price'] * 100, 2) if (
|
||||
prev_city_stat and prev_city_stat.get('avg_price')) else None
|
||||
city_yoy = round((city_price - ly_city_stat['avg_price']) / ly_city_stat['avg_price'] * 100, 2) if (
|
||||
ly_city_stat and ly_city_stat.get('avg_price')) else None
|
||||
|
||||
# 写入MONGODB
|
||||
action = UpdateOne({'city': city, 'cat': 'city', 'date': year_month},
|
||||
{'$set': {'city': city, 'cat': 'city', 'date': year_month, 'avg_price': int(city_price),
|
||||
'mom': city_mom, 'yoy': city_yoy,
|
||||
'updated_date': datetime.now()}},
|
||||
upsert=True)
|
||||
|
||||
@@ -60,21 +76,33 @@ def calc(city, year_month):
|
||||
for region in areautils.get_all_districts(city):
|
||||
region_avg_prices = region_median_count_prices['median']
|
||||
region_count = region_median_count_prices['count']
|
||||
prev_district_stat = office_sell_stat_conn.find_one(
|
||||
{'city': city, 'cat': 'district', 'date': {'$lte': last_month}, 'district': region, 'avg_price': {'$gt': 0}},
|
||||
sort=[('date', -1)]
|
||||
)
|
||||
if region in region_avg_prices and region_count[region] >= 10:
|
||||
avg_price = region_avg_prices[region]
|
||||
avg_price = int(region_avg_prices[region])
|
||||
else:
|
||||
# 处理均价为空情况,用上月均价替代
|
||||
district_stat = office_sell_stat_conn.find_one({'city': city, 'cat': 'district', 'date': last_month, 'district': region})
|
||||
if pd.isna(district_stat):
|
||||
if pd.isna(prev_district_stat):
|
||||
continue
|
||||
last_month_region_price = district_stat['avg_price']
|
||||
last_month_region_price = prev_district_stat['avg_price']
|
||||
if pd.isna(last_month_region_price):
|
||||
continue
|
||||
avg_price = last_month_region_price
|
||||
avg_price = int(last_month_region_price)
|
||||
|
||||
ly_district_stat = office_sell_stat_conn.find_one(
|
||||
{'city': city, 'cat': 'district', 'date': ly_month, 'district': region, 'avg_price': {'$gt': 0}}
|
||||
)
|
||||
region_mom = round((avg_price - prev_district_stat['avg_price']) / prev_district_stat['avg_price'] * 100, 2) if (
|
||||
prev_district_stat and prev_district_stat.get('avg_price')) else None
|
||||
region_yoy = round((avg_price - ly_district_stat['avg_price']) / ly_district_stat['avg_price'] * 100, 2) if (
|
||||
ly_district_stat and ly_district_stat.get('avg_price')) else None
|
||||
|
||||
action = UpdateOne({'city': city, 'cat': 'district', 'date': year_month, 'district': region},
|
||||
{'$set': {'city': city, 'cat': 'district', 'date': year_month, 'district': region,
|
||||
'avg_price': int(avg_price), 'updated_date': datetime.now()}},
|
||||
'avg_price': int(avg_price), 'mom': region_mom, 'yoy': region_yoy,
|
||||
'updated_date': datetime.now()}},
|
||||
upsert=True)
|
||||
actions.append(action)
|
||||
office_sell_stat_conn.bulk_write(actions)
|
||||
|
||||
Reference in New Issue
Block a user