From 8bf59f7b50af9d27fd0f67e871260795e852c73e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 22 Jan 2025 14:25:22 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E4=BB=B7=E6=A0=BCNAN?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/office/rent/price.py | 61 +++++++++++++++++++++++++++------------ 1 file changed, 43 insertions(+), 18 deletions(-) diff --git a/dags/office/rent/price.py b/dags/office/rent/price.py index 844bbea..d55b08c 100644 --- a/dags/office/rent/price.py +++ b/dags/office/rent/price.py @@ -19,18 +19,32 @@ def calc(city, year_month): office = es_client.scan_by_sql_to_data_frame( "select * from latest_seawater where cat='office_rent' and city='" + city + "' and p_date>=" + str( year_month) + "01 and p_date<=" + str(year_month) + "31 ") - if len(office) == 0: - return - office['day_avg_price'] = office['avg_price'] / 30 - # 计算总价的Z-score - office['Z_Score'] = (office['day_avg_price'] - office['day_avg_price'].mean()) / office['day_avg_price'].std() - # 剔除Z-score超过2的异常值 - office_cleaned = office[np.abs(office['Z_Score']) < 0.1] - city_price = round(office_cleaned['day_avg_price'].mean(), 2) - region_avg_prices = office_cleaned.groupby(office_cleaned['region'])['day_avg_price'].mean() + if len(office) > 0: + office['day_avg_price'] = office['avg_price'] / 30 + # 计算总价的Z-score + office['Z_Score'] = (office['day_avg_price'] - office['day_avg_price'].mean()) / office['day_avg_price'].std() + # 剔除Z-score超过0.1的异常值 + office_cleaned = office[np.abs(office['Z_Score']) < 0.1] + city_price = round(office_cleaned['day_avg_price'].mean(), 2) + region_avg_prices = office_cleaned.groupby(office_cleaned['region'])['day_avg_price'].mean() + else: + city_price = None + region_avg_prices = {} actions = list() + last_month = int( + dateutils.format_date(dateutils.add_months(dateutils.parse_date(str(year_month), '%Y%m'), months=-1), '%Y%m')) + if pd.isna(city_price): + # 处理均价为空情况,用上月均价替代 + stat = office_rent_stat_conn.find_one({'city': city, 'cat': 'city', 'date': last_month}) + if pd.isna(stat): + return False + last_month_city_price = stat['avg_price'] + if pd.isna(last_month_city_price): + return False + city_price = last_month_city_price + # 写入MONGODB action = UpdateOne({'city': city, 'cat': 'city', 'date': year_month}, {'$set': {'city': city, 'cat': 'city', 'date': year_month, 'avg_price': city_price, @@ -39,6 +53,17 @@ def calc(city, year_month): actions.append(action) for region, avg_price in region_avg_prices.items(): + # 不会出现NaN的情况, 因为聚合来的 + #if pd.isna(avg_price): + # # 处理均价为空情况,用上月均价替代 + # district_stat = office_rent_stat_conn.find_one({'city': city, 'cat': 'district', 'date': last_month, 'district': region}) + # if pd.isna(district_stat): + # continue + # last_month_region_price = district_stat['avg_price'] + # if pd.isna(last_month_region_price): + # continue + # avg_price = last_month_region_price + action = UpdateOne({'city': city, 'cat': 'district', 'date': year_month, 'district': region}, {'$set': {'city': city, 'cat': 'district', 'date': year_month, 'district': region, 'avg_price': round(avg_price, 2), 'updated_date': datetime.now()}}, @@ -47,8 +72,8 @@ def calc(city, year_month): office_rent_stat_conn.bulk_write(actions) log.info(f"{city} {year_month} 均价计算完毕!") -yearmonth = sys.argv[1] if len(sys.argv) > 1 else '' +yearmonth = sys.argv[1] if len(sys.argv) > 1 else '' if yearmonth == '': date = dateutils.add_months(datetime.today(), months=-1) @@ -59,8 +84,8 @@ try: except Exception as e: log.error(e) -# 初始化跑两年 -# 创建从2024年1月到2024年12月的日期范围 +# # 初始化跑两年 +# # # 创建从2024年1月到2024年12月的日期范围 # date_range = pd.date_range(start='2023-01-01', end='2024-12-31', freq='M') # # # 创建一个包含这些日期的DataFrame @@ -70,10 +95,10 @@ except Exception as e: # df.set_index('date', inplace=True) # # # 遍历DataFrame -# for date, row in df.iterrows(): -# date_format = int(dateutils.format_date(date, '%Y%m')) -# try: -# for city in areautils.get_all_cities(): +# for city in areautils.get_all_cities(): +# for date, row in df.iterrows(): +# try: +# date_format = int(dateutils.format_date(date, '%Y%m')) # calc(city, date_format) -# except Exception as e: -# log.error(e) +# except Exception as e: +# log.error(e)