Files
airflow/dags/office/sell/fill_base_price.py
T

132 lines
6.1 KiB
Python

import pandas as pd
from datetime import datetime
from pymongo import UpdateOne
import logging
log = logging.getLogger(__name__)
def fill_base_price(**kwargs):
"""
写字楼基准价
"""
from goojia_utils import dateutils, MAIN_MONGO_CLIENT
office_sell_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office_sell_stat')
office_rent_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office_rent_stat')
# 从 Airflow 的 dag_run.conf 获取参数
conf = kwargs.get('dag_run').conf if kwargs.get('dag_run') else {}
city = conf.get('city', '')
year_month = conf.get('yearmonth', '')
district = conf.get('district', '')
office_name = conf.get('name', '')
if year_month == "":
year_month = dateutils.format_date(dateutils.add_months(datetime.today(), months=-1), "%Y%m")
query = {'cat': 'office'}
if year_month != '':
query['date'] = year_month
if city != '':
query['city'] = city
if district != '' and office_name != '':
query['district'] = district
query['name'] = office_name
office_sell = pd.DataFrame(list(office_sell_stat_conn.find(query)))
office_rent = pd.DataFrame(list(office_rent_stat_conn.find(query)))
if len(office_rent) <= 0:
return
if year_month != '':
calc_by_yearmonth(office_sell, office_rent, year_month, office_sell_stat_conn)
else:
# 对于单个重算的不进行补充,已经自己补充过了
pass
def calc_by_yearmonth(office_sell, office_rent, year_month, office_sell_stat_conn=None):
from goojia_utils import MAIN_MONGO_CLIENT, threads
if office_sell_stat_conn is None:
office_sell_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office_sell_stat')
office_sell_df = office_sell[['city', 'district', 'name', 'avg_price']]
office_sell_df = office_sell_df.rename(columns={'avg_price': 'sell_price'})
office_rent_df = office_rent[['city', 'district', 'name', 'avg_price', 'estate_type2', 'bed_kind', 'location']]
office_rent_df = office_rent_df.rename(columns={'avg_price': 'rent_price'})
# 每月动态计算系数,因为不同月份会有差异
merge_df = pd.merge(office_sell_df, office_rent_df, how='left', on=['city', 'district', 'name'])
merge_df = merge_df[(merge_df['rent_price'].notna()) & (merge_df['sell_price'].notna())]
merge_df['ratio'] = merge_df['sell_price'] / (merge_df['rent_price'] * 30 * 12)
merge_df['district_ratio'] = merge_df.groupby(by=['city', 'district'])['ratio'].transform('median')
merge_df['city_ratio'] = merge_df.groupby(by=['city'])['ratio'].transform('median')
district_radio = dict(zip(merge_df["city"] + "_" + merge_df["district"], merge_df["district_ratio"]))
city_radio = dict(zip(merge_df["city"], merge_df["city_ratio"]))
merge2_df = pd.merge(office_rent_df, office_sell_df, how='left', on=['city', 'district', 'name'])
merge2_df = merge2_df[(merge2_df['rent_price'].notna()) & (merge2_df['sell_price'].isna())]
def deal(item):
"""
填充存在租金,不存在售价的数据
"""
if item['city'] + "_" + item['district'] in district_radio.keys():
item['sell_price'] = round(
district_radio[item['city'] + "_" + item['district']] * item['rent_price'] * 30 * 12, 0)
elif item['city'] in city_radio.keys():
item['sell_price'] = round(city_radio[item['city']] * item['rent_price'] * 30 * 12, 0)
return item
if len(merge2_df) == 0:
return
merge2_df = threads.process_dataframe_multithread(merge2_df, deal, 10)
merge2_df = merge2_df[merge2_df['sell_price'].notna()]
curr_date = int(year_month)
year = curr_date // 100
month = curr_date % 100
prev_date = (year - 1) * 100 + 12 if month == 1 else curr_date - 1
ly_date = curr_date - 100
proj = {'_id': 0, 'city': 1, 'district': 1, 'name': 1, 'avg_price': 1}
prev_map = {(d.get('city'), d.get('district'), d.get('name')): d.get('avg_price') for d in office_sell_stat_conn.find({'cat': 'office', 'date': prev_date}, proj) if d.get('avg_price')}
ly_map = {(d.get('city'), d.get('district'), d.get('name')): d.get('avg_price') for d in office_sell_stat_conn.find({'cat': 'office', 'date': ly_date}, proj) if d.get('avg_price')}
actions = list()
count = 0
for index, row in merge2_df.iterrows():
curr_city = row['city']
region = row['district']
name = row['name']
sell_price = int(row['sell_price'])
estate_type2 = row['estate_type2']
bed_kind = row['bed_kind']
location = row['location']
prev_p = prev_map.get((curr_city, region, name))
ly_p = ly_map.get((curr_city, region, name))
mom = round((sell_price - prev_p) / prev_p * 100, 2) if prev_p else None
yoy = round((sell_price - ly_p) / ly_p * 100, 2) if ly_p else None
action = UpdateOne({'cat': 'office', 'city': curr_city, 'district': region, 'name': name, 'date': curr_date},
{'$set': {'city': curr_city, 'cat': 'office', 'date': curr_date, 'district': region,
'name': name,
'avg_price': sell_price, 'mom': mom, 'yoy': yoy,
'estate_type2': estate_type2, 'bed_kind': bed_kind,
'location': location,
'calc_type': 5, 'updated_date': datetime.now()}},
upsert=True)
actions.append(action)
if len(actions) == 5000:
office_sell_stat_conn.bulk_write(actions)
count += len(actions)
print("已处理{}条数据".format(count))
actions.clear()
if len(actions) > 0:
office_sell_stat_conn.bulk_write(actions)
count += len(actions)
print("已处理{}条数据".format(count))
print("处理完毕")
if __name__ == "__main__":
# 测试脚本:直接运行时模拟 Airflow 的上下文
fill_base_price(dag_run=type('DagRun', (), {'conf': {'city': '上海', 'district': '静安', 'name': '凯迪克大厦'}}))