Files
airflow/dags/mt/supply/rent_avg_price.py
T
2022-07-01 16:13:46 +08:00

92 lines
3.8 KiB
Python

"""
租金、均价导出
"""
from goojia_utils import MAIN_MONGO_CLIENT, dateutils, commutils
import pandas as pd
import datetime
import logging
import sys
district_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_stat')
sqmetre_rent_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'sqmetre_rent')
city_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/美团19城市列表.xlsx')
cities = city_df['城市列表'].values.tolist()
path = "/home/airflow/data/mt"
output_date = sys.argv[1] if sys.argv[1] else dateutils.format_date(datetime.datetime.now(), '%Y%m')
seaweed_new_result_df = pd.read_csv(path + '/FJW_COMM_' + output_date + '.csv', sep='@', encoding='utf-8')
month_start = dateutils.format_date(dateutils.add_months(dateutils.parse_date(output_date, '%Y%m'), months=-1), '%Y%m') + '01'
curr_month_start = output_date + '01'
logging.info("日期信息:" + month_start + ' ' + curr_month_start)
month_start_date = dateutils.parse_date(month_start, '%Y%m%d')
curr_month_start_date = dateutils.parse_date(curr_month_start, '%Y%m%d')
threshold_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/租金和房价设置阀值(19个城市).xlsx')
min_rent = dict()
max_rent = dict()
min_price = dict()
max_price = dict()
for index, item in threshold_df.iterrows():
if item['租金最小值'] != '缺租金':
min_rent[item['城市']] = item['租金最小值']
if item['租金最大值'] != '缺租金':
max_rent[item['城市']] = item['租金最大值']
min_price[item['城市']] = item['均价最小值']
max_price[item['城市']] = item['均价最大值']
avg_df = pd.DataFrame()
for city in cities:
# 求当月均价均值
temp_df = pd.DataFrame(list(district_stat_conn.find(
{'weekend': {'$gt': month_start_date, '$lte': curr_month_start_date}, 'city': city},
{'city': 1, 'region': 1, 'district_name': 1, 'avg_price': 1, 'weekend': 1}).batch_size(10000)))
temp_df['avg_price_mean'] = temp_df.groupby(by=['city', 'region', 'district_name'])['avg_price'].transform('mean').astype(int)
temp_df = temp_df[['city', 'region', 'district_name', 'avg_price_mean']]
temp_df = temp_df.drop_duplicates(subset=['city', 'region', 'district_name'], keep='first')
# 过滤阈值
if city in min_price:
temp_df = temp_df[(temp_df['avg_price_mean'] >= min_price[city]) & (temp_df['avg_price_mean'] <= max_price[city])]
avg_df = pd.concat([avg_df, temp_df])
avg_df = avg_df.rename(columns={'district_name': 'name', 'avg_price_mean': 'avg_price'})
rent_df = pd.DataFrame()
for city in cities:
# 求租金
temp_df = pd.DataFrame(list(sqmetre_rent_conn.find(
{'weekend': {'$gt': month_start_date, '$lte': curr_month_start_date}, 'city': city},
{'city': 1, 'region': 1, 'district_name': 1, 'avg_price': 1}).batch_size(10000)))
# 根据阈值过滤
if city in min_rent:
temp_df = temp_df[(temp_df['avg_price'] >= min_rent[city]) & (temp_df['avg_price'] <= max_rent[city])]
rent_df = pd.concat([rent_df, temp_df])
rent_df = rent_df.rename(columns={'district_name': 'name', 'avg_price': 'rent_price'})
avg_df = avg_df[['city', 'region', 'name', 'avg_price']]
rent_df = rent_df[['city', 'region', 'name', 'rent_price']]
result_df = pd.merge(seaweed_new_result_df, avg_df, how='left', on=['city', 'region', 'name'])
result_df = pd.merge(result_df, rent_df, how='left', on=['city', 'region', 'name'])
def format_rent(x):
"""
过滤租售比,大于2000租金为空
:param x:
:return:
"""
x.rent_price = commutils.format_rent_price(x.avg_price, x.rent_price, 1500)
return x
result_df = result_df.apply(lambda x: format_rent(x), axis=1)
result_df.to_csv(path + '/FJW_COMM_' + output_date + '.csv', sep='@', encoding='utf-8', index=False)