122 lines
5.2 KiB
Python
122 lines
5.2 KiB
Python
"""
|
|
租金、均价导出
|
|
"""
|
|
|
|
from goojia_utils import MAIN_MONGO_CLIENT, dateutils, commutils, ClickHouseClient
|
|
import pandas as pd
|
|
import datetime
|
|
import logging
|
|
import sys
|
|
|
|
#district_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_stat')
|
|
# 月度均价表
|
|
district_sell_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_sell_stat')
|
|
sqmetre_rent_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'sqmetre_rent')
|
|
tmp_ck = ClickHouseClient(database='tmp')
|
|
|
|
|
|
city_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/美团88城市列表.xlsx')
|
|
cities = city_df['城市列表'].values.tolist()
|
|
|
|
path = "/home/airflow/data/mt"
|
|
|
|
output_date = sys.argv[1] if len(sys.argv) > 1 else dateutils.format_date(datetime.datetime.now(), '%Y%m')
|
|
|
|
seaweed_new_result_df = tmp_ck.read_sql_query("select * from mt_community")
|
|
del seaweed_new_result_df['m_date']
|
|
|
|
if 'avg_price' in seaweed_new_result_df:
|
|
seaweed_new_result_df = seaweed_new_result_df.drop(columns=['avg_price', 'rent_price'])
|
|
|
|
month_start = dateutils.format_date(dateutils.add_months(dateutils.parse_date(output_date, '%Y%m'), months=-1), '%Y%m') + '01'
|
|
year_month = month_start[0:6]
|
|
curr_month_start = output_date + '01'
|
|
logging.info("日期信息:" + month_start + ' ' + curr_month_start)
|
|
|
|
month_start_date = dateutils.parse_date(month_start, '%Y%m%d')
|
|
curr_month_start_date = dateutils.parse_date(curr_month_start, '%Y%m%d')
|
|
|
|
threshold_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/租金和房价设置阀值.xlsx')
|
|
min_rent = dict()
|
|
max_rent = dict()
|
|
min_price = dict()
|
|
max_price = dict()
|
|
for index, item in threshold_df.iterrows():
|
|
if item['租金最小值'] != '缺租金':
|
|
min_rent[item['城市']] = item['租金最小值']
|
|
if item['租金最大值'] != '缺租金':
|
|
max_rent[item['城市']] = item['租金最大值']
|
|
min_price[item['城市']] = item['均价最小值']
|
|
max_price[item['城市']] = item['均价最大值']
|
|
|
|
# 已过期 20221108 改为从月度均价表获取【该表已控制每月涨跌幅】
|
|
# avg_df = pd.DataFrame()
|
|
# for city in cities:
|
|
# # 求当月均价均值
|
|
# temp_df = pd.DataFrame(list(district_stat_conn.find(
|
|
# {'weekend': {'$gt': month_start_date, '$lte': curr_month_start_date}, 'city': city},
|
|
# {'city': 1, 'region': 1, 'district_name': 1, 'avg_price': 1, 'weekend': 1}).batch_size(10000)))
|
|
# temp_df['avg_price_mean'] = temp_df.groupby(by=['city', 'region', 'district_name'])['avg_price'].transform('mean').astype(int)
|
|
# temp_df = temp_df[['city', 'region', 'district_name', 'avg_price_mean']]
|
|
# temp_df = temp_df.drop_duplicates(subset=['city', 'region', 'district_name'], keep='first')
|
|
# # 过滤阈值
|
|
# if city in min_price:
|
|
# temp_df = temp_df[(temp_df['avg_price_mean'] >= min_price[city]) & (temp_df['avg_price_mean'] <= max_price[city])]
|
|
# avg_df = pd.concat([avg_df, temp_df])
|
|
# avg_df = avg_df.rename(columns={'district_name': 'name', 'avg_price_mean': 'avg_price'})
|
|
|
|
avg_df = pd.DataFrame(list(district_sell_conn.find({'date': int(year_month)}, {'city': 1, 'region': 1, 'district_name': 1, 'avg_price': 1}).batch_size(100000)))
|
|
avg_df = avg_df.rename(columns={'district_name': 'name'})
|
|
|
|
rent_df = pd.DataFrame()
|
|
for city in cities:
|
|
# 求租金
|
|
temp_df = pd.DataFrame(list(sqmetre_rent_conn.find(
|
|
{'weekend': {'$gt': month_start_date, '$lte': curr_month_start_date}, 'city': city, 'cat': 'district'},
|
|
{'city': 1, 'region': 1, 'district_name': 1, 'avg_price': 1}).batch_size(10000)))
|
|
# 根据阈值过滤
|
|
if city in min_rent:
|
|
temp_df = temp_df[(temp_df['avg_price'] >= min_rent[city]) & (temp_df['avg_price'] <= max_rent[city])]
|
|
rent_df = pd.concat([rent_df, temp_df])
|
|
|
|
rent_df = rent_df.rename(columns={'district_name': 'name', 'avg_price': 'rent_price'})
|
|
|
|
avg_df = avg_df[['city', 'region', 'name', 'avg_price']]
|
|
rent_df = rent_df[['city', 'region', 'name', 'rent_price']]
|
|
|
|
result_df = pd.merge(seaweed_new_result_df, avg_df, how='left', on=['city', 'region', 'name'])
|
|
result_df = pd.merge(result_df, rent_df, how='left', on=['city', 'region', 'name'])
|
|
|
|
none_rent_and_price_df = pd.read_excel('/opt/airflow/dags/repo/dags/mt/supply/data/特殊城市既无租金又无价格小区过滤.xlsx')
|
|
none_rent_and_price_cities = none_rent_and_price_df['city'].values.tolist()
|
|
|
|
|
|
def format_rent(x):
|
|
"""
|
|
过滤租售比,大于2000租金为空
|
|
:param x:
|
|
:return:
|
|
"""
|
|
# 哈尔滨不过滤最低租售比
|
|
if '哈尔滨' == x['city']:
|
|
x.rent_price = commutils.format_rent_price(x.avg_price, x.rent_price, 1500, 0)
|
|
else:
|
|
x.rent_price = commutils.format_rent_price(x.avg_price, x.rent_price, 1500, 100)
|
|
# 过滤租金与均价无的小区
|
|
if len(none_rent_and_price_cities) > 0:
|
|
if x['city'] in none_rent_and_price_cities and (('rent_price' not in x or str(x['rent_price']) == 'nan') and ('avg_price' not in x or str(x['avg_price']) == 'nan')):
|
|
x['ignore_comm'] = 1
|
|
return x
|
|
|
|
|
|
result_df['ignore_comm'] = None
|
|
result_df = result_df.apply(lambda x: format_rent(x), axis=1)
|
|
result_df = result_df[result_df['ignore_comm'] != 1]
|
|
|
|
del result_df['ignore_comm']
|
|
|
|
ck_delete_sql = "ALTER TABLE tmp.mt_community DELETE WHERE 1=1 "
|
|
tmp_ck.execute(ck_delete_sql)
|
|
|
|
tmp_ck.to_sql_plus(result_df, 'mt_community')
|