From 2d266a30c266726251c040d3498835bc8d42cdaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=96=87=E8=89=B3?= Date: Mon, 19 Oct 2026 11:54:35 +0800 Subject: [PATCH] feat(office): auto-calculate mom and yoy for city, district, and office monthly stats --- dags/office/rent/base_price.py | 44 ++++++++++++++++++++++++ dags/office/rent/price.py | 52 ++++++++++++++++++++--------- dags/office/sell/base_price.py | 44 ++++++++++++++++++++++++ dags/office/sell/fill_base_price.py | 22 +++++++++--- dags/office/sell/fill_price.py | 27 +++++++++++++-- dags/office/sell/price.py | 50 +++++++++++++++++++++------ 6 files changed, 206 insertions(+), 33 deletions(-) diff --git a/dags/office/rent/base_price.py b/dags/office/rent/base_price.py index 5561ab6..170ed0f 100644 --- a/dags/office/rent/base_price.py +++ b/dags/office/rent/base_price.py @@ -186,6 +186,49 @@ def calculate_mom_yoy_for_office(city, district, office_name): office_rent_stat_conn.bulk_write(actions) +def calculate_mom_yoy_by_month(year_month, city=None): + """ + 批量计算某月所有写字楼的 mom (环比) 和 yoy (同比) 并更新 + """ + curr_date = int(year_month) + year = curr_date // 100 + month = curr_date % 100 + prev_date = (year - 1) * 100 + 12 if month == 1 else curr_date - 1 + ly_date = curr_date - 100 + + base_query = {'cat': 'office'} + if pd.notna(city) and city != '': + base_query['city'] = city + + proj = {'_id': 1, 'city': 1, 'district': 1, 'name': 1, 'avg_price': 1} + curr_docs = list(office_rent_stat_conn.find({**base_query, 'date': curr_date}, proj)) + if not curr_docs: + return + + prev_docs = list(office_rent_stat_conn.find({**base_query, 'date': prev_date}, proj)) + ly_docs = list(office_rent_stat_conn.find({**base_query, 'date': ly_date}, proj)) + + prev_map = {(d.get('city'), d.get('district'), d.get('name')): d.get('avg_price') for d in prev_docs if d.get('avg_price')} + ly_map = {(d.get('city'), d.get('district'), d.get('name')): d.get('avg_price') for d in ly_docs if d.get('avg_price')} + + actions = [] + for doc in curr_docs: + curr_price = doc.get('avg_price') + if not curr_price: + continue + key = (doc.get('city'), doc.get('district'), doc.get('name')) + prev_price = prev_map.get(key) + ly_price = ly_map.get(key) + mom = round((curr_price - prev_price) / prev_price * 100, 2) if prev_price else None + yoy = round((curr_price - ly_price) / ly_price * 100, 2) if ly_price else None + actions.append(UpdateOne({'_id': doc['_id']}, {'$set': {'mom': mom, 'yoy': yoy}})) + if len(actions) == 5000: + office_rent_stat_conn.bulk_write(actions) + actions.clear() + if actions: + office_rent_stat_conn.bulk_write(actions) + + def calcByCity(city, year_month): """ 写字楼基准价 @@ -212,6 +255,7 @@ def calcByCity(city, year_month): office = getOffice(city) update_office(city, office, sample, year_month) + calculate_mom_yoy_by_month(year_month, city) def update_office(city, office, sample, year_month): diff --git a/dags/office/rent/price.py b/dags/office/rent/price.py index 8f3691a..eb156fe 100644 --- a/dags/office/rent/price.py +++ b/dags/office/rent/price.py @@ -37,25 +37,36 @@ def calc(city, year_month): region_median_count_prices = {'median': {}, 'count': {}} actions = list() - + year_month = int(year_month) last_month = int( dateutils.format_date(dateutils.add_months(dateutils.parse_date(str(year_month), '%Y%m'), months=-1), '%Y%m')) + ly_month = year_month - 100 + + prev_city_stat = office_rent_stat_conn.find_one( + {'city': city, 'cat': 'city', 'date': {'$lte': int(last_month)}, 'avg_price': {'$lte': 15, '$gt': 0}}, + sort=[('date', -1)] + ) if pd.isna(city_price): # 处理均价为空情况,用上月均价替代 - stat = office_rent_stat_conn.find_one( - {'city': city, 'cat': 'city', 'date': {'$lte': int(last_month)}, 'avg_price': {'$lte': 15, '$gt': 0}}, - sort=[('date', -1)] - ) - if pd.isna(stat): + if pd.isna(prev_city_stat): return False - last_month_city_price = stat['avg_price'] + last_month_city_price = prev_city_stat['avg_price'] if pd.isna(last_month_city_price): return False city_price = last_month_city_price + ly_city_stat = office_rent_stat_conn.find_one( + {'city': city, 'cat': 'city', 'date': int(ly_month), 'avg_price': {'$lte': 15, '$gt': 0}} + ) + city_mom = round((city_price - prev_city_stat['avg_price']) / prev_city_stat['avg_price'] * 100, 2) if ( + prev_city_stat and prev_city_stat.get('avg_price')) else None + city_yoy = round((city_price - ly_city_stat['avg_price']) / ly_city_stat['avg_price'] * 100, 2) if ( + ly_city_stat and ly_city_stat.get('avg_price')) else None + # 写入MONGODB action = UpdateOne({'city': city, 'cat': 'city', 'date': int(year_month)}, {'$set': {'city': city, 'cat': 'city', 'date': int(year_month), 'avg_price': city_price, + 'mom': city_mom, 'yoy': city_yoy, 'updated_date': datetime.now()}}, upsert=True) @@ -63,24 +74,33 @@ def calc(city, year_month): for region in areautils.get_all_districts(city): region_avg_prices = region_median_count_prices['median'] region_count = region_median_count_prices['count'] + prev_district_stat = office_rent_stat_conn.find_one( + {'city': city, 'cat': 'district', 'date': {'$lte': int(last_month)}, 'district': region, 'avg_price': {'$lte': 15, '$gt': 0}}, + sort=[('date', -1)] + ) if region in region_avg_prices and region_count[region] >= 10: - avg_price = region_avg_prices[region] + avg_price = round(region_avg_prices[region], 2) else: # 处理均价为空情况,用上月均价替代 - district_stat = office_rent_stat_conn.find_one( - {'city': city, 'cat': 'district', 'date': {'$lte': int(last_month)}, 'district': region, 'avg_price': {'$lte': 15, '$gt': 0}}, - sort=[('date', -1)] - ) - if pd.isna(district_stat): + if pd.isna(prev_district_stat): continue - last_month_region_price = district_stat['avg_price'] + last_month_region_price = prev_district_stat['avg_price'] if pd.isna(last_month_region_price): continue - avg_price = last_month_region_price + avg_price = round(last_month_region_price, 2) + + ly_district_stat = office_rent_stat_conn.find_one( + {'city': city, 'cat': 'district', 'date': int(ly_month), 'district': region, 'avg_price': {'$lte': 15, '$gt': 0}} + ) + region_mom = round((avg_price - prev_district_stat['avg_price']) / prev_district_stat['avg_price'] * 100, 2) if ( + prev_district_stat and prev_district_stat.get('avg_price')) else None + region_yoy = round((avg_price - ly_district_stat['avg_price']) / ly_district_stat['avg_price'] * 100, 2) if ( + ly_district_stat and ly_district_stat.get('avg_price')) else None action = UpdateOne({'city': city, 'cat': 'district', 'date': int(year_month), 'district': region}, {'$set': {'city': city, 'cat': 'district', 'date': int(year_month), 'district': region, - 'avg_price': round(avg_price, 2), 'updated_date': datetime.now()}}, + 'avg_price': avg_price, 'mom': region_mom, 'yoy': region_yoy, + 'updated_date': datetime.now()}}, upsert=True) actions.append(action) office_rent_stat_conn.bulk_write(actions) diff --git a/dags/office/sell/base_price.py b/dags/office/sell/base_price.py index 95a2ddf..054f747 100644 --- a/dags/office/sell/base_price.py +++ b/dags/office/sell/base_price.py @@ -187,6 +187,49 @@ def calculate_mom_yoy_for_office(city, district, office_name): office_sell_stat_conn.bulk_write(actions) +def calculate_mom_yoy_by_month(year_month, city=None): + """ + 批量计算某月所有写字楼出售的 mom (环比) 和 yoy (同比) 并更新 + """ + curr_date = int(year_month) + year = curr_date // 100 + month = curr_date % 100 + prev_date = (year - 1) * 100 + 12 if month == 1 else curr_date - 1 + ly_date = curr_date - 100 + + base_query = {'cat': 'office'} + if pd.notna(city) and city != '': + base_query['city'] = city + + proj = {'_id': 1, 'city': 1, 'district': 1, 'name': 1, 'avg_price': 1} + curr_docs = list(office_sell_stat_conn.find({**base_query, 'date': curr_date}, proj)) + if not curr_docs: + return + + prev_docs = list(office_sell_stat_conn.find({**base_query, 'date': prev_date}, proj)) + ly_docs = list(office_sell_stat_conn.find({**base_query, 'date': ly_date}, proj)) + + prev_map = {(d.get('city'), d.get('district'), d.get('name')): d.get('avg_price') for d in prev_docs if d.get('avg_price')} + ly_map = {(d.get('city'), d.get('district'), d.get('name')): d.get('avg_price') for d in ly_docs if d.get('avg_price')} + + actions = [] + for doc in curr_docs: + curr_price = doc.get('avg_price') + if not curr_price: + continue + key = (doc.get('city'), doc.get('district'), doc.get('name')) + prev_price = prev_map.get(key) + ly_price = ly_map.get(key) + mom = round((curr_price - prev_price) / prev_price * 100, 2) if prev_price else None + yoy = round((curr_price - ly_price) / ly_price * 100, 2) if ly_price else None + actions.append(UpdateOne({'_id': doc['_id']}, {'$set': {'mom': mom, 'yoy': yoy}})) + if len(actions) == 5000: + office_sell_stat_conn.bulk_write(actions) + actions.clear() + if actions: + office_sell_stat_conn.bulk_write(actions) + + def calcByCity(city, year_month): """ 写字楼基准价 @@ -214,6 +257,7 @@ def calcByCity(city, year_month): office = getOffice(city) update_office(city, office, sample, year_month) + calculate_mom_yoy_by_month(year_month, city) def update_office(city, office, sample, year_month): diff --git a/dags/office/sell/fill_base_price.py b/dags/office/sell/fill_base_price.py index 36214aa..3940912 100644 --- a/dags/office/sell/fill_base_price.py +++ b/dags/office/sell/fill_base_price.py @@ -82,20 +82,34 @@ def calc_by_yearmonth(office_sell, office_rent, year_month, office_sell_stat_con merge2_df = threads.process_dataframe_multithread(merge2_df, deal, 10) merge2_df = merge2_df[merge2_df['sell_price'].notna()] + curr_date = int(year_month) + year = curr_date // 100 + month = curr_date % 100 + prev_date = (year - 1) * 100 + 12 if month == 1 else curr_date - 1 + ly_date = curr_date - 100 + proj = {'_id': 0, 'city': 1, 'district': 1, 'name': 1, 'avg_price': 1} + prev_map = {(d.get('city'), d.get('district'), d.get('name')): d.get('avg_price') for d in office_sell_stat_conn.find({'cat': 'office', 'date': prev_date}, proj) if d.get('avg_price')} + ly_map = {(d.get('city'), d.get('district'), d.get('name')): d.get('avg_price') for d in office_sell_stat_conn.find({'cat': 'office', 'date': ly_date}, proj) if d.get('avg_price')} + actions = list() count = 0 for index, row in merge2_df.iterrows(): curr_city = row['city'] region = row['district'] name = row['name'] - sell_price = row['sell_price'] + sell_price = int(row['sell_price']) estate_type2 = row['estate_type2'] bed_kind = row['bed_kind'] location = row['location'] - action = UpdateOne({'cat': 'office', 'city': curr_city, 'district': region, 'name': name, 'date': year_month}, - {'$set': {'city': curr_city, 'cat': 'office', 'date': year_month, 'district': region, + prev_p = prev_map.get((curr_city, region, name)) + ly_p = ly_map.get((curr_city, region, name)) + mom = round((sell_price - prev_p) / prev_p * 100, 2) if prev_p else None + yoy = round((sell_price - ly_p) / ly_p * 100, 2) if ly_p else None + action = UpdateOne({'cat': 'office', 'city': curr_city, 'district': region, 'name': name, 'date': curr_date}, + {'$set': {'city': curr_city, 'cat': 'office', 'date': curr_date, 'district': region, 'name': name, - 'avg_price': int(sell_price), 'estate_type2': estate_type2, 'bed_kind': bed_kind, + 'avg_price': sell_price, 'mom': mom, 'yoy': yoy, + 'estate_type2': estate_type2, 'bed_kind': bed_kind, 'location': location, 'calc_type': 5, 'updated_date': datetime.now()}}, upsert=True) diff --git a/dags/office/sell/fill_price.py b/dags/office/sell/fill_price.py index c3695b3..a142d85 100644 --- a/dags/office/sell/fill_price.py +++ b/dags/office/sell/fill_price.py @@ -59,11 +59,25 @@ def calc(city, year_month): merge2_df = threads.process_dataframe_multithread(merge2_df, deal_city, 10) merge2_df = merge2_df[merge2_df['sell_price'].notna()] + year = year_month // 100 + month = year_month % 100 + prev_date = (year - 1) * 100 + 12 if month == 1 else year_month - 1 + ly_date = year_month - 100 + + prev_city_map = {d.get('city'): d.get('avg_price') for d in office_sell_stat_conn.find({'date': prev_date, 'cat': 'city'}) if d.get('avg_price')} + ly_city_map = {d.get('city'): d.get('avg_price') for d in office_sell_stat_conn.find({'date': ly_date, 'cat': 'city'}) if d.get('avg_price')} + actions = list() # 写入MONGODB for index, row in merge2_df.iterrows(): + sell_p = int(row['sell_price']) + prev_p = prev_city_map.get(row['city']) + ly_p = ly_city_map.get(row['city']) + mom = round((sell_p - prev_p) / prev_p * 100, 2) if prev_p else None + yoy = round((sell_p - ly_p) / ly_p * 100, 2) if ly_p else None action = UpdateOne({'city': row['city'], 'cat': 'city', 'date': year_month}, - {'$set': {'city': row['city'], 'cat': 'city', 'date': year_month, 'calc_type': 2, 'avg_price': int(row['sell_price']), + {'$set': {'city': row['city'], 'cat': 'city', 'date': year_month, 'calc_type': 2, 'avg_price': sell_p, + 'mom': mom, 'yoy': yoy, 'updated_date': datetime.now()}}, upsert=True) actions.append(action) @@ -89,10 +103,19 @@ def calc(city, year_month): merge2_df = threads.process_dataframe_multithread(merge2_df, deal_district, 10) merge2_df = merge2_df[merge2_df['sell_price'].notna()] + prev_dist_map = {(d.get('city'), d.get('district')): d.get('avg_price') for d in office_sell_stat_conn.find({'date': prev_date, 'cat': 'district'}) if d.get('avg_price')} + ly_dist_map = {(d.get('city'), d.get('district')): d.get('avg_price') for d in office_sell_stat_conn.find({'date': ly_date, 'cat': 'district'}) if d.get('avg_price')} + # 写入MONGODB for index, row in merge2_df.iterrows(): + sell_p = int(row['sell_price']) + prev_p = prev_dist_map.get((row['city'], row['district'])) + ly_p = ly_dist_map.get((row['city'], row['district'])) + mom = round((sell_p - prev_p) / prev_p * 100, 2) if prev_p else None + yoy = round((sell_p - ly_p) / ly_p * 100, 2) if ly_p else None action = UpdateOne({'city': row['city'], 'district': row['district'], 'cat': 'district', 'date': year_month}, - {'$set': {'city': row['city'], 'district': row['district'], 'cat': 'district', 'date': year_month, 'calc_type': 2, 'avg_price': int(row['sell_price']), + {'$set': {'city': row['city'], 'district': row['district'], 'cat': 'district', 'date': year_month, 'calc_type': 2, 'avg_price': sell_p, + 'mom': mom, 'yoy': yoy, 'updated_date': datetime.now()}}, upsert=True) actions.append(action) diff --git a/dags/office/sell/price.py b/dags/office/sell/price.py index f2149b3..4ef43fd 100644 --- a/dags/office/sell/price.py +++ b/dags/office/sell/price.py @@ -37,22 +37,38 @@ def calc(city, year_month): region_median_count_prices = {'median': {}, 'count': {}} actions = list() - + year_month = int(year_month) last_month = int( dateutils.format_date(dateutils.add_months(dateutils.parse_date(str(year_month), '%Y%m'), months=-1), '%Y%m')) + ly_month = year_month - 100 + + prev_city_stat = office_sell_stat_conn.find_one( + {'city': city, 'cat': 'city', 'date': {'$lte': last_month}, 'avg_price': {'$gt': 0}}, + sort=[('date', -1)] + ) if pd.isna(city_price): # 处理均价为空情况,用上月均价替代 - stat = office_sell_stat_conn.find_one({'city': city, 'cat': 'city', 'date': last_month}) - if pd.isna(stat): + if pd.isna(prev_city_stat): return False - last_month_city_price = stat['avg_price'] + last_month_city_price = prev_city_stat['avg_price'] if pd.isna(last_month_city_price): return False - city_price = last_month_city_price + city_price = int(last_month_city_price) + else: + city_price = int(city_price) + + ly_city_stat = office_sell_stat_conn.find_one( + {'city': city, 'cat': 'city', 'date': ly_month, 'avg_price': {'$gt': 0}} + ) + city_mom = round((city_price - prev_city_stat['avg_price']) / prev_city_stat['avg_price'] * 100, 2) if ( + prev_city_stat and prev_city_stat.get('avg_price')) else None + city_yoy = round((city_price - ly_city_stat['avg_price']) / ly_city_stat['avg_price'] * 100, 2) if ( + ly_city_stat and ly_city_stat.get('avg_price')) else None # 写入MONGODB action = UpdateOne({'city': city, 'cat': 'city', 'date': year_month}, {'$set': {'city': city, 'cat': 'city', 'date': year_month, 'avg_price': int(city_price), + 'mom': city_mom, 'yoy': city_yoy, 'updated_date': datetime.now()}}, upsert=True) @@ -60,21 +76,33 @@ def calc(city, year_month): for region in areautils.get_all_districts(city): region_avg_prices = region_median_count_prices['median'] region_count = region_median_count_prices['count'] + prev_district_stat = office_sell_stat_conn.find_one( + {'city': city, 'cat': 'district', 'date': {'$lte': last_month}, 'district': region, 'avg_price': {'$gt': 0}}, + sort=[('date', -1)] + ) if region in region_avg_prices and region_count[region] >= 10: - avg_price = region_avg_prices[region] + avg_price = int(region_avg_prices[region]) else: # 处理均价为空情况,用上月均价替代 - district_stat = office_sell_stat_conn.find_one({'city': city, 'cat': 'district', 'date': last_month, 'district': region}) - if pd.isna(district_stat): + if pd.isna(prev_district_stat): continue - last_month_region_price = district_stat['avg_price'] + last_month_region_price = prev_district_stat['avg_price'] if pd.isna(last_month_region_price): continue - avg_price = last_month_region_price + avg_price = int(last_month_region_price) + + ly_district_stat = office_sell_stat_conn.find_one( + {'city': city, 'cat': 'district', 'date': ly_month, 'district': region, 'avg_price': {'$gt': 0}} + ) + region_mom = round((avg_price - prev_district_stat['avg_price']) / prev_district_stat['avg_price'] * 100, 2) if ( + prev_district_stat and prev_district_stat.get('avg_price')) else None + region_yoy = round((avg_price - ly_district_stat['avg_price']) / ly_district_stat['avg_price'] * 100, 2) if ( + ly_district_stat and ly_district_stat.get('avg_price')) else None action = UpdateOne({'city': city, 'cat': 'district', 'date': year_month, 'district': region}, {'$set': {'city': city, 'cat': 'district', 'date': year_month, 'district': region, - 'avg_price': int(avg_price), 'updated_date': datetime.now()}}, + 'avg_price': int(avg_price), 'mom': region_mom, 'yoy': region_yoy, + 'updated_date': datetime.now()}}, upsert=True) actions.append(action) office_sell_stat_conn.bulk_write(actions)