Files
airflow/dags/office/rent/price.py
T

140 lines
6.2 KiB
Python

import sys
from goojia_utils import ElasticSearchClient, MAIN_MONGO_CLIENT, dateutils, areautils
import numpy as np
from pymongo import UpdateOne
from datetime import datetime
import logging
import pandas as pd
log = logging.getLogger(__name__)
es_client = ElasticSearchClient(host='192.168.0.206')
office_rent_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office_rent_stat')
def calc(city, year_month):
if len(str(year_month)) != 6:
raise Exception('年月格式错误yyyyMM')
office = es_client.scan_by_sql_to_data_frame(
"select * from latest_seawater where cat='office_rent' and city='" + city + "' and p_date>=" + str(
year_month) + "01 and p_date<=" + str(year_month) + "31 and avg_price>=30 and avg_price<=450 ")
if len(office) > 0:
office['day_avg_price'] = office['avg_price'] / 30
# 计算总价的Z-score
office['Z_Score'] = (office['day_avg_price'] - office['day_avg_price'].mean()) / office['day_avg_price'].std()
# 剔除Z-score超过0.1的异常值
office_cleaned = office[np.abs(office['Z_Score']) < 0.1]
# 至少样本数大于等于10
if len(office_cleaned['day_avg_price']) >= 10:
city_price = round(office_cleaned['day_avg_price'].median(), 2)
region_median_count_prices = office_cleaned.groupby(office_cleaned['region'])['day_avg_price'].agg(['median', 'count']).to_dict()
else:
city_price = None
region_median_count_prices = {'median': {}, 'count': {}}
else:
city_price = None
region_median_count_prices = {'median': {}, 'count': {}}
actions = list()
year_month = int(year_month)
last_month = int(
dateutils.format_date(dateutils.add_months(dateutils.parse_date(str(year_month), '%Y%m'), months=-1), '%Y%m'))
ly_month = year_month - 100
prev_city_stat = office_rent_stat_conn.find_one(
{'city': city, 'cat': 'city', 'date': {'$lte': int(last_month)}, 'avg_price': {'$lte': 15, '$gt': 0}},
sort=[('date', -1)]
)
if pd.isna(city_price):
# 处理均价为空情况,用上月均价替代
if pd.isna(prev_city_stat):
return False
last_month_city_price = prev_city_stat['avg_price']
if pd.isna(last_month_city_price):
return False
city_price = last_month_city_price
ly_city_stat = office_rent_stat_conn.find_one(
{'city': city, 'cat': 'city', 'date': int(ly_month), 'avg_price': {'$lte': 15, '$gt': 0}}
)
city_mom = round((city_price - prev_city_stat['avg_price']) / prev_city_stat['avg_price'] * 100, 2) if (
prev_city_stat and prev_city_stat.get('avg_price')) else None
city_yoy = round((city_price - ly_city_stat['avg_price']) / ly_city_stat['avg_price'] * 100, 2) if (
ly_city_stat and ly_city_stat.get('avg_price')) else None
# 写入MONGODB
action = UpdateOne({'city': city, 'cat': 'city', 'date': int(year_month)},
{'$set': {'city': city, 'cat': 'city', 'date': int(year_month), 'avg_price': city_price,
'mom': city_mom, 'yoy': city_yoy,
'updated_date': datetime.now()}},
upsert=True)
actions.append(action)
for region in areautils.get_all_districts(city):
region_avg_prices = region_median_count_prices['median']
region_count = region_median_count_prices['count']
prev_district_stat = office_rent_stat_conn.find_one(
{'city': city, 'cat': 'district', 'date': {'$lte': int(last_month)}, 'district': region, 'avg_price': {'$lte': 15, '$gt': 0}},
sort=[('date', -1)]
)
if region in region_avg_prices and region_count[region] >= 10:
avg_price = round(region_avg_prices[region], 2)
else:
# 处理均价为空情况,用上月均价替代
if pd.isna(prev_district_stat):
continue
last_month_region_price = prev_district_stat['avg_price']
if pd.isna(last_month_region_price):
continue
avg_price = round(last_month_region_price, 2)
ly_district_stat = office_rent_stat_conn.find_one(
{'city': city, 'cat': 'district', 'date': int(ly_month), 'district': region, 'avg_price': {'$lte': 15, '$gt': 0}}
)
region_mom = round((avg_price - prev_district_stat['avg_price']) / prev_district_stat['avg_price'] * 100, 2) if (
prev_district_stat and prev_district_stat.get('avg_price')) else None
region_yoy = round((avg_price - ly_district_stat['avg_price']) / ly_district_stat['avg_price'] * 100, 2) if (
ly_district_stat and ly_district_stat.get('avg_price')) else None
action = UpdateOne({'city': city, 'cat': 'district', 'date': int(year_month), 'district': region},
{'$set': {'city': city, 'cat': 'district', 'date': int(year_month), 'district': region,
'avg_price': avg_price, 'mom': region_mom, 'yoy': region_yoy,
'updated_date': datetime.now()}},
upsert=True)
actions.append(action)
office_rent_stat_conn.bulk_write(actions)
log.info(f"{city} {year_month} 均价计算完毕!")
if __name__ == "__main__":
yearmonth = sys.argv[1] if len(sys.argv) > 1 else ''
if yearmonth == '':
date = dateutils.add_months(datetime.today(), months=-1)
yearmonth = int(dateutils.format_date(date, '%Y%m'))
try:
for city in areautils.get_all_cities():
calc(city, yearmonth)
except Exception as e:
log.error(e)
# # 初始化从2024年9月
# 创建从2024年9月到2024年12月的日期范围
# date_range = pd.date_range(start='2024-09-01', end='2025-01-31', freq='M')
#
# # 创建一个包含这些日期的DataFrame
# df = pd.DataFrame({'date': date_range, 'value': range(1, len(date_range) + 1)})
#
# # 将日期设置为索引
# df.set_index('date', inplace=True)
#
# # 遍历DataFrame
# for city in areautils.get_all_cities():
# for date, row in df.iterrows():
# try:
# date_format = int(dateutils.format_date(date, '%Y%m'))
# calc(city, date_format)
# except Exception as e:
# log.error(e)