新增写字楼售价填充
This commit is contained in:
@@ -9,7 +9,7 @@ from airflow import DAG
|
||||
|
||||
with DAG(
|
||||
dag_id="office_rent_base_price_dag",
|
||||
schedule_interval="0 1 27 * *",
|
||||
schedule_interval="0 1 26 * *",
|
||||
start_date=pendulum.datetime(2025, 2, 28, tz="Asia/Shanghai"),
|
||||
catchup=True,
|
||||
description="写字楼基准价计算"
|
||||
|
||||
@@ -9,7 +9,7 @@ from airflow import DAG
|
||||
|
||||
with DAG(
|
||||
dag_id="office_sell_base_price_dag",
|
||||
schedule_interval="0 1 26 * *",
|
||||
schedule_interval="0 1 27 * *",
|
||||
start_date=pendulum.datetime(2025, 3, 24, tz="Asia/Shanghai"),
|
||||
catchup=True,
|
||||
description="写字楼出售基准价计算"
|
||||
@@ -18,5 +18,9 @@ with DAG(
|
||||
task_id='sell_base_price',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/office/sell/base_price.py {{dag_run.conf["yearmonth"] if "yearmonth" in dag_run.conf}}',
|
||||
)
|
||||
t1
|
||||
t2 = BashOperator(
|
||||
task_id='fill_base_price',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/office/sell/fill_base_price.py {{dag_run.conf["yearmonth"] if "yearmonth" in dag_run.conf}}',
|
||||
)
|
||||
t1 >> t2
|
||||
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
from goojia_utils import DIDATA_DEV_ELASTIC_CLIENT, dateutils, MAIN_MONGO_CLIENT, threads, areautils
|
||||
import pandas as pd
|
||||
from datetime import datetime
|
||||
from pymongo import UpdateOne
|
||||
import sys
|
||||
|
||||
office_sell_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office_sell_stat')
|
||||
office_rent_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office_rent_stat')
|
||||
|
||||
def calc(city, year_month):
|
||||
query = {'date': year_month, 'cat': 'office'}
|
||||
if pd.notna(city):
|
||||
query['city'] = city
|
||||
office_sell = pd.DataFrame(list(office_sell_stat_conn.find(query)))
|
||||
office_rent = pd.DataFrame(list(office_rent_stat_conn.find(query)))
|
||||
|
||||
if len(office_rent) <= 0:
|
||||
return
|
||||
office_sell_df = office_sell[['city', 'district', 'name', 'avg_price']]
|
||||
office_sell_df = office_sell_df.rename(columns={'avg_price': 'sell_price'})
|
||||
office_rent_df = office_rent[['city', 'district', 'name', 'avg_price', 'estate_type2', 'bed_kind','location']]
|
||||
office_rent_df = office_rent_df.rename(columns={'avg_price': 'rent_price'})
|
||||
# 每月动态计算系数,因为不同月份会有差异
|
||||
merge_df = pd.merge(office_sell_df, office_rent_df, how='left', on=['city', 'district', 'name'])
|
||||
merge_df = merge_df[(merge_df['rent_price'].notna()) & (merge_df['sell_price'].notna())]
|
||||
|
||||
merge_df['ratio'] = merge_df['sell_price'] / (merge_df['rent_price'] * 30 * 12)
|
||||
merge_df['district_ratio'] = merge_df.groupby(by=['city', 'district'])['ratio'].transform('median')
|
||||
merge_df['city_ratio'] = merge_df.groupby(by=['city'])['ratio'].transform('median')
|
||||
|
||||
district_radio = dict(zip(merge_df["city"] + "_" + merge_df["district"], merge_df["district_ratio"]))
|
||||
city_radio = dict(zip(merge_df["city"], merge_df["city_ratio"]))
|
||||
|
||||
merge2_df = pd.merge(office_rent_df, office_sell_df, how='left', on=['city', 'district', 'name'])
|
||||
merge2_df = merge2_df[(merge2_df['rent_price'].notna()) & (merge2_df['sell_price'].isna())]
|
||||
|
||||
def deal(item):
|
||||
"""
|
||||
填充存在租金,不存在售价的数据
|
||||
"""
|
||||
if item['city'] + "_" + item['district'] in district_radio.keys():
|
||||
item['sell_price'] = round(district_radio[item['city'] + "_" + item['district']] * item['rent_price'] * 30 * 12, 0)
|
||||
elif item['city'] in city_radio.keys():
|
||||
item['sell_price'] = round(city_radio[item['city']] * item['rent_price'] * 30 * 12, 0)
|
||||
return item
|
||||
|
||||
merge2_df = threads.process_dataframe_multithread(merge2_df, deal, 10)
|
||||
merge2_df = merge2_df[merge2_df['sell_price'].notna()]
|
||||
|
||||
actions = list()
|
||||
count = 0
|
||||
for index, row in merge2_df.iterrows():
|
||||
curr_city = row['city']
|
||||
region = row['district']
|
||||
name = row['name']
|
||||
sell_price = row['sell_price']
|
||||
estate_type2 = row['estate_type2']
|
||||
bed_kind = row['bed_kind']
|
||||
location = row['location']
|
||||
action = UpdateOne({'cat': 'office', 'city': curr_city, 'district': region, 'name': name, 'date': year_month},
|
||||
{'$set': {'city': curr_city, 'cat': 'office', 'date': year_month, 'district': region,
|
||||
'name': name,
|
||||
'avg_price': sell_price, 'estate_type2': estate_type2, 'bed_kind': bed_kind,
|
||||
'location': location,
|
||||
'calc_type': 5, 'updated_date': datetime.now()}},
|
||||
upsert=True)
|
||||
actions.append(action)
|
||||
if len(actions) == 5000:
|
||||
office_sell_stat_conn.bulk_write(actions)
|
||||
count += len(actions)
|
||||
print("已处理{}条数据".format(count))
|
||||
actions.clear()
|
||||
if len(actions) > 0:
|
||||
office_sell_stat_conn.bulk_write(actions)
|
||||
count += len(actions)
|
||||
print("已处理{}条数据".format(count))
|
||||
print("处理完毕")
|
||||
|
||||
|
||||
yearmonth = sys.argv[1] if len(sys.argv) > 1 else ''
|
||||
|
||||
if yearmonth == '':
|
||||
date = dateutils.add_months(datetime.today(), months=-1)
|
||||
yearmonth = int(dateutils.format_date(date, '%Y%m'))
|
||||
try:
|
||||
if type(yearmonth) is str:
|
||||
yearmonth = int(yearmonth)
|
||||
# for curr_city in areautils.get_all_cities():
|
||||
# print("正在处理城市:{}".format(curr_city))
|
||||
calc(None, yearmonth)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
|
||||
Reference in New Issue
Block a user