修复价格NAN问题
This commit is contained in:
+43
-18
@@ -19,18 +19,32 @@ def calc(city, year_month):
|
||||
office = es_client.scan_by_sql_to_data_frame(
|
||||
"select * from latest_seawater where cat='office_rent' and city='" + city + "' and p_date>=" + str(
|
||||
year_month) + "01 and p_date<=" + str(year_month) + "31 ")
|
||||
if len(office) == 0:
|
||||
return
|
||||
office['day_avg_price'] = office['avg_price'] / 30
|
||||
# 计算总价的Z-score
|
||||
office['Z_Score'] = (office['day_avg_price'] - office['day_avg_price'].mean()) / office['day_avg_price'].std()
|
||||
# 剔除Z-score超过2的异常值
|
||||
office_cleaned = office[np.abs(office['Z_Score']) < 0.1]
|
||||
city_price = round(office_cleaned['day_avg_price'].mean(), 2)
|
||||
region_avg_prices = office_cleaned.groupby(office_cleaned['region'])['day_avg_price'].mean()
|
||||
if len(office) > 0:
|
||||
office['day_avg_price'] = office['avg_price'] / 30
|
||||
# 计算总价的Z-score
|
||||
office['Z_Score'] = (office['day_avg_price'] - office['day_avg_price'].mean()) / office['day_avg_price'].std()
|
||||
# 剔除Z-score超过0.1的异常值
|
||||
office_cleaned = office[np.abs(office['Z_Score']) < 0.1]
|
||||
city_price = round(office_cleaned['day_avg_price'].mean(), 2)
|
||||
region_avg_prices = office_cleaned.groupby(office_cleaned['region'])['day_avg_price'].mean()
|
||||
else:
|
||||
city_price = None
|
||||
region_avg_prices = {}
|
||||
|
||||
actions = list()
|
||||
|
||||
last_month = int(
|
||||
dateutils.format_date(dateutils.add_months(dateutils.parse_date(str(year_month), '%Y%m'), months=-1), '%Y%m'))
|
||||
if pd.isna(city_price):
|
||||
# 处理均价为空情况,用上月均价替代
|
||||
stat = office_rent_stat_conn.find_one({'city': city, 'cat': 'city', 'date': last_month})
|
||||
if pd.isna(stat):
|
||||
return False
|
||||
last_month_city_price = stat['avg_price']
|
||||
if pd.isna(last_month_city_price):
|
||||
return False
|
||||
city_price = last_month_city_price
|
||||
|
||||
# 写入MONGODB
|
||||
action = UpdateOne({'city': city, 'cat': 'city', 'date': year_month},
|
||||
{'$set': {'city': city, 'cat': 'city', 'date': year_month, 'avg_price': city_price,
|
||||
@@ -39,6 +53,17 @@ def calc(city, year_month):
|
||||
|
||||
actions.append(action)
|
||||
for region, avg_price in region_avg_prices.items():
|
||||
# 不会出现NaN的情况, 因为聚合来的
|
||||
#if pd.isna(avg_price):
|
||||
# # 处理均价为空情况,用上月均价替代
|
||||
# district_stat = office_rent_stat_conn.find_one({'city': city, 'cat': 'district', 'date': last_month, 'district': region})
|
||||
# if pd.isna(district_stat):
|
||||
# continue
|
||||
# last_month_region_price = district_stat['avg_price']
|
||||
# if pd.isna(last_month_region_price):
|
||||
# continue
|
||||
# avg_price = last_month_region_price
|
||||
|
||||
action = UpdateOne({'city': city, 'cat': 'district', 'date': year_month, 'district': region},
|
||||
{'$set': {'city': city, 'cat': 'district', 'date': year_month, 'district': region,
|
||||
'avg_price': round(avg_price, 2), 'updated_date': datetime.now()}},
|
||||
@@ -47,8 +72,8 @@ def calc(city, year_month):
|
||||
office_rent_stat_conn.bulk_write(actions)
|
||||
log.info(f"{city} {year_month} 均价计算完毕!")
|
||||
|
||||
yearmonth = sys.argv[1] if len(sys.argv) > 1 else ''
|
||||
|
||||
yearmonth = sys.argv[1] if len(sys.argv) > 1 else ''
|
||||
|
||||
if yearmonth == '':
|
||||
date = dateutils.add_months(datetime.today(), months=-1)
|
||||
@@ -59,8 +84,8 @@ try:
|
||||
except Exception as e:
|
||||
log.error(e)
|
||||
|
||||
# 初始化跑两年
|
||||
# 创建从2024年1月到2024年12月的日期范围
|
||||
# # 初始化跑两年
|
||||
# # # 创建从2024年1月到2024年12月的日期范围
|
||||
# date_range = pd.date_range(start='2023-01-01', end='2024-12-31', freq='M')
|
||||
#
|
||||
# # 创建一个包含这些日期的DataFrame
|
||||
@@ -70,10 +95,10 @@ except Exception as e:
|
||||
# df.set_index('date', inplace=True)
|
||||
#
|
||||
# # 遍历DataFrame
|
||||
# for date, row in df.iterrows():
|
||||
# date_format = int(dateutils.format_date(date, '%Y%m'))
|
||||
# try:
|
||||
# for city in areautils.get_all_cities():
|
||||
# for city in areautils.get_all_cities():
|
||||
# for date, row in df.iterrows():
|
||||
# try:
|
||||
# date_format = int(dateutils.format_date(date, '%Y%m'))
|
||||
# calc(city, date_format)
|
||||
# except Exception as e:
|
||||
# log.error(e)
|
||||
# except Exception as e:
|
||||
# log.error(e)
|
||||
|
||||
Reference in New Issue
Block a user