132 lines
6.1 KiB
Python
132 lines
6.1 KiB
Python
import pandas as pd
|
|
from datetime import datetime
|
|
from pymongo import UpdateOne
|
|
import logging
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
def fill_base_price(**kwargs):
|
|
"""
|
|
写字楼基准价
|
|
"""
|
|
from goojia_utils import dateutils, MAIN_MONGO_CLIENT
|
|
|
|
office_sell_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office_sell_stat')
|
|
office_rent_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office_rent_stat')
|
|
|
|
# 从 Airflow 的 dag_run.conf 获取参数
|
|
conf = kwargs.get('dag_run').conf if kwargs.get('dag_run') else {}
|
|
city = conf.get('city', '')
|
|
year_month = conf.get('yearmonth', '')
|
|
district = conf.get('district', '')
|
|
office_name = conf.get('name', '')
|
|
if year_month == "":
|
|
year_month = dateutils.format_date(dateutils.add_months(datetime.today(), months=-1), "%Y%m")
|
|
|
|
query = {'cat': 'office'}
|
|
if year_month != '':
|
|
query['date'] = year_month
|
|
if city != '':
|
|
query['city'] = city
|
|
if district != '' and office_name != '':
|
|
query['district'] = district
|
|
query['name'] = office_name
|
|
office_sell = pd.DataFrame(list(office_sell_stat_conn.find(query)))
|
|
office_rent = pd.DataFrame(list(office_rent_stat_conn.find(query)))
|
|
|
|
if len(office_rent) <= 0:
|
|
return
|
|
if year_month != '':
|
|
calc_by_yearmonth(office_sell, office_rent, year_month, office_sell_stat_conn)
|
|
else:
|
|
# 对于单个重算的不进行补充,已经自己补充过了
|
|
pass
|
|
|
|
def calc_by_yearmonth(office_sell, office_rent, year_month, office_sell_stat_conn=None):
|
|
from goojia_utils import MAIN_MONGO_CLIENT, threads
|
|
|
|
if office_sell_stat_conn is None:
|
|
office_sell_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office_sell_stat')
|
|
office_sell_df = office_sell[['city', 'district', 'name', 'avg_price']]
|
|
office_sell_df = office_sell_df.rename(columns={'avg_price': 'sell_price'})
|
|
office_rent_df = office_rent[['city', 'district', 'name', 'avg_price', 'estate_type2', 'bed_kind', 'location']]
|
|
office_rent_df = office_rent_df.rename(columns={'avg_price': 'rent_price'})
|
|
# 每月动态计算系数,因为不同月份会有差异
|
|
merge_df = pd.merge(office_sell_df, office_rent_df, how='left', on=['city', 'district', 'name'])
|
|
merge_df = merge_df[(merge_df['rent_price'].notna()) & (merge_df['sell_price'].notna())]
|
|
|
|
merge_df['ratio'] = merge_df['sell_price'] / (merge_df['rent_price'] * 30 * 12)
|
|
merge_df['district_ratio'] = merge_df.groupby(by=['city', 'district'])['ratio'].transform('median')
|
|
merge_df['city_ratio'] = merge_df.groupby(by=['city'])['ratio'].transform('median')
|
|
|
|
district_radio = dict(zip(merge_df["city"] + "_" + merge_df["district"], merge_df["district_ratio"]))
|
|
city_radio = dict(zip(merge_df["city"], merge_df["city_ratio"]))
|
|
|
|
merge2_df = pd.merge(office_rent_df, office_sell_df, how='left', on=['city', 'district', 'name'])
|
|
merge2_df = merge2_df[(merge2_df['rent_price'].notna()) & (merge2_df['sell_price'].isna())]
|
|
|
|
def deal(item):
|
|
"""
|
|
填充存在租金,不存在售价的数据
|
|
"""
|
|
if item['city'] + "_" + item['district'] in district_radio.keys():
|
|
item['sell_price'] = round(
|
|
district_radio[item['city'] + "_" + item['district']] * item['rent_price'] * 30 * 12, 0)
|
|
elif item['city'] in city_radio.keys():
|
|
item['sell_price'] = round(city_radio[item['city']] * item['rent_price'] * 30 * 12, 0)
|
|
return item
|
|
|
|
if len(merge2_df) == 0:
|
|
return
|
|
merge2_df = threads.process_dataframe_multithread(merge2_df, deal, 10)
|
|
merge2_df = merge2_df[merge2_df['sell_price'].notna()]
|
|
|
|
curr_date = int(year_month)
|
|
year = curr_date // 100
|
|
month = curr_date % 100
|
|
prev_date = (year - 1) * 100 + 12 if month == 1 else curr_date - 1
|
|
ly_date = curr_date - 100
|
|
proj = {'_id': 0, 'city': 1, 'district': 1, 'name': 1, 'avg_price': 1}
|
|
prev_map = {(d.get('city'), d.get('district'), d.get('name')): d.get('avg_price') for d in office_sell_stat_conn.find({'cat': 'office', 'date': prev_date}, proj) if d.get('avg_price')}
|
|
ly_map = {(d.get('city'), d.get('district'), d.get('name')): d.get('avg_price') for d in office_sell_stat_conn.find({'cat': 'office', 'date': ly_date}, proj) if d.get('avg_price')}
|
|
|
|
actions = list()
|
|
count = 0
|
|
for index, row in merge2_df.iterrows():
|
|
curr_city = row['city']
|
|
region = row['district']
|
|
name = row['name']
|
|
sell_price = int(row['sell_price'])
|
|
estate_type2 = row['estate_type2']
|
|
bed_kind = row['bed_kind']
|
|
location = row['location']
|
|
prev_p = prev_map.get((curr_city, region, name))
|
|
ly_p = ly_map.get((curr_city, region, name))
|
|
mom = round((sell_price - prev_p) / prev_p * 100, 2) if prev_p else None
|
|
yoy = round((sell_price - ly_p) / ly_p * 100, 2) if ly_p else None
|
|
action = UpdateOne({'cat': 'office', 'city': curr_city, 'district': region, 'name': name, 'date': curr_date},
|
|
{'$set': {'city': curr_city, 'cat': 'office', 'date': curr_date, 'district': region,
|
|
'name': name,
|
|
'avg_price': sell_price, 'mom': mom, 'yoy': yoy,
|
|
'estate_type2': estate_type2, 'bed_kind': bed_kind,
|
|
'location': location,
|
|
'calc_type': 5, 'updated_date': datetime.now()}},
|
|
upsert=True)
|
|
actions.append(action)
|
|
if len(actions) == 5000:
|
|
office_sell_stat_conn.bulk_write(actions)
|
|
count += len(actions)
|
|
print("已处理{}条数据".format(count))
|
|
actions.clear()
|
|
if len(actions) > 0:
|
|
office_sell_stat_conn.bulk_write(actions)
|
|
count += len(actions)
|
|
print("已处理{}条数据".format(count))
|
|
print("处理完毕")
|
|
|
|
if __name__ == "__main__":
|
|
# 测试脚本:直接运行时模拟 Airflow 的上下文
|
|
fill_base_price(dag_run=type('DagRun', (), {'conf': {'city': '上海', 'district': '静安', 'name': '凯迪克大厦'}}))
|
|
|