109 lines
3.6 KiB
Python
109 lines
3.6 KiB
Python
from airflow.providers.standard.operators.bash import BashOperator
|
|
from airflow.operators.python import PythonOperator, BranchPythonOperator
|
|
from airflow.operators.empty import EmptyOperator
|
|
import pendulum
|
|
|
|
from airflow import DAG
|
|
|
|
|
|
def run_base_price_calc(**kwargs):
|
|
from office.sell.base_price import calc
|
|
return calc(**kwargs)
|
|
|
|
|
|
def run_fill_base_price(**kwargs):
|
|
from office.sell.fill_base_price import fill_base_price
|
|
return fill_base_price(**kwargs)
|
|
|
|
|
|
def choose_task(**context):
|
|
"""根据 dag_run.conf 决定执行哪个任务"""
|
|
conf = context['dag_run'].conf or {}
|
|
task_type = conf.get('task_type', 'both') # 默认执行所有任务
|
|
|
|
if task_type == 'office':
|
|
# 只执行写字楼基础价格计算 (t1 -> t2)
|
|
return 'office_sell_base_price_calc'
|
|
elif task_type == 'city_region':
|
|
# 只执行城市区域价格计算 (t3 -> t4)
|
|
return 'office_city_region_sell_base_price_calc'
|
|
else:
|
|
# both 或其他情况: 执行所有任务 (t1 -> t2 -> t3 -> t4)
|
|
return 'office_sell_base_price_calc'
|
|
|
|
|
|
def choose_after_office(**context):
|
|
"""t2 执行完后判断是否继续到 t3"""
|
|
conf = context['dag_run'].conf or {}
|
|
task_type = conf.get('task_type', 'both')
|
|
|
|
if task_type == 'both':
|
|
# both 模式继续执行 t3
|
|
return 'office_city_region_sell_base_price_calc'
|
|
else:
|
|
# office 模式结束,不执行 t3
|
|
return 'skip_city_region'
|
|
|
|
|
|
with DAG(
|
|
dag_id="office_sell_dag",
|
|
# 每月25日3点运行
|
|
schedule="0 3 25 * *",
|
|
start_date=pendulum.datetime(2025, 1, 1, tz="Asia/Shanghai"),
|
|
catchup=False,
|
|
description="写字楼出售价格计算",
|
|
tags=["office", "sell"],
|
|
) as dag:
|
|
# 第一个分支任务:选择起始任务
|
|
branch_task = BranchPythonOperator(
|
|
task_id='choose_branch',
|
|
python_callable=choose_task,
|
|
# 🛡️ 已安全移除 provide_context=True
|
|
)
|
|
|
|
# 任务1: 写字楼出售基础价格计算
|
|
t1 = PythonOperator(
|
|
task_id='office_sell_base_price_calc',
|
|
python_callable=run_base_price_calc,
|
|
# 🛡️ 已安全移除 provide_context=True
|
|
)
|
|
|
|
# 任务2: 无样本写字楼出售基础价格填充
|
|
t2 = PythonOperator(
|
|
task_id='office_no_sample_sell_base_price_calc',
|
|
python_callable=run_fill_base_price,
|
|
# 🛡️ 已安全移除 provide_context=True
|
|
)
|
|
|
|
# 第二个分支任务:t2 后判断是否继续
|
|
branch_after_office = BranchPythonOperator(
|
|
task_id='choose_after_office',
|
|
python_callable=choose_after_office,
|
|
# 🛡️ 已安全移除 provide_context=True
|
|
)
|
|
|
|
# 跳过标记(office 模式结束点)
|
|
skip_city_region = EmptyOperator(
|
|
task_id='skip_city_region'
|
|
)
|
|
|
|
# 任务3: 城市区域写字楼出售价格计算
|
|
t3 = BashOperator(
|
|
task_id='office_city_region_sell_base_price_calc',
|
|
bash_command='python /opt/airflow/dags/repo/dags/office/sell/price.py {{ dag_run.conf.get("yearmonth", "") }} ',
|
|
)
|
|
|
|
# 任务4: 城市区域无样本写字楼出售价格填充
|
|
t4 = BashOperator(
|
|
task_id='office_city_region_sell_no_sample_base_price_calc',
|
|
bash_command='python /opt/airflow/dags/repo/dags/office/sell/fill_price.py {{ dag_run.conf.get("yearmonth", "") }}',
|
|
)
|
|
|
|
# 定义任务依赖关系
|
|
# office 路径: branch -> t1 -> t2 -> branch_after_office -> skip_city_region
|
|
# city_region 路径: branch -> t3 -> t4
|
|
# both 路径: branch -> t1 -> t2 -> branch_after_office -> t3 -> t4
|
|
|
|
branch_task >> [t1, t3]
|
|
t1 >> t2 >> branch_after_office >> [t3, skip_city_region]
|
|
t3 >> t4 |