From 5918e5c3d1436041b4a22b45bc13672812ff5301 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Tue, 9 Aug 2022 17:13:48 +0800 Subject: [PATCH] =?UTF-8?q?=E5=90=88=E5=B9=B6=E5=91=A8=E8=BE=B9=E9=85=8D?= =?UTF-8?q?=E5=A5=97dag?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/example_python_operator.py | 17 -------- dags/poi/business/district_facilities.py | 41 +++++++++++++++++++ dags/poi/district_bus_dag.py | 26 ------------ ...bank_dag.py => district_facilities_dag.py} | 8 ++-- dags/poi/district_hospital_dag.py | 22 ---------- dags/poi/district_school_dag.py | 22 ---------- dags/poi/district_supermarket_dag.py | 22 ---------- 7 files changed, 45 insertions(+), 113 deletions(-) delete mode 100644 dags/example_python_operator.py create mode 100644 dags/poi/business/district_facilities.py delete mode 100644 dags/poi/district_bus_dag.py rename dags/poi/{district_bank_dag.py => district_facilities_dag.py} (67%) delete mode 100644 dags/poi/district_hospital_dag.py delete mode 100644 dags/poi/district_school_dag.py delete mode 100644 dags/poi/district_supermarket_dag.py diff --git a/dags/example_python_operator.py b/dags/example_python_operator.py deleted file mode 100644 index b68252c..0000000 --- a/dags/example_python_operator.py +++ /dev/null @@ -1,17 +0,0 @@ -import logging -import shutil -import time -from datetime import datetime -from airflow import DAG -from airflow.decorators import task - -log = logging.getLogger(__name__) - -with DAG( - dag_id='example_python_operator', - schedule_interval="*/1 * * * *", - start_date=datetime(2021, 1, 1), - catchup=False, - tags=['example'], -) as dag: - print('success') \ No newline at end of file diff --git a/dags/poi/business/district_facilities.py b/dags/poi/business/district_facilities.py new file mode 100644 index 0000000..18d929e --- /dev/null +++ b/dags/poi/business/district_facilities.py @@ -0,0 +1,41 @@ +import sys + +type = 'all' +if len(sys.argv) > 1: + type = sys.argv[1] + +print("输出类型" + type) + +if type is not None: + if type == 'bank': + import district_bank as bank + bank + elif type == 'bus': + import district_bus as bus + bus + elif type == 'hospital': + import district_hospital as hospital + hospital + elif type == 'school': + import district_school as school + school + elif type == 'subway': + import district_subway as subway + subway + elif type == 'supermarket': + import district_supermarket as supermarket + supermarket + else: + import district_bank as bank + import district_bus as bus + import district_hospital as hospital + import district_school as school + import district_subway as subway + import district_supermarket as supermarket + + bank + bus + hospital + school + subway + supermarket diff --git a/dags/poi/district_bus_dag.py b/dags/poi/district_bus_dag.py deleted file mode 100644 index 5edb8ab..0000000 --- a/dags/poi/district_bus_dag.py +++ /dev/null @@ -1,26 +0,0 @@ -""" -小区周边公交DAG -""" -from airflow.operators.bash import BashOperator - -import pendulum - -from airflow import DAG - -with DAG( - "district_bus_dag", - # 每月1日9点运行 - schedule_interval="0 9 1 * *", - start_date=pendulum.datetime(2022, 6, 18, tz="Asia/Shanghai"), - catchup=True, -) as dag: - t1 = BashOperator( - task_id='district_subway', - bash_command='python /opt/airflow/dags/repo/dags/poi/business/district_subway.py', - ) - t2 = BashOperator( - task_id='district_bus', - bash_command='python /opt/airflow/dags/repo/dags/poi/business/district_bus.py', - ) - t1 >> t2 - diff --git a/dags/poi/district_bank_dag.py b/dags/poi/district_facilities_dag.py similarity index 67% rename from dags/poi/district_bank_dag.py rename to dags/poi/district_facilities_dag.py index 427e663..54ce1d5 100644 --- a/dags/poi/district_bank_dag.py +++ b/dags/poi/district_facilities_dag.py @@ -1,5 +1,5 @@ """ -小区周边银行DAG +小区周边配套DAG """ from airflow.operators.bash import BashOperator @@ -8,15 +8,15 @@ import pendulum from airflow import DAG with DAG( - "district_bank_dag", + "district_facilities_dag", # 每月1日1点运行 schedule_interval="0 1 1 * *", start_date=pendulum.datetime(2022, 6, 18, tz="Asia/Shanghai"), catchup=True, ) as dag: t1 = BashOperator( - task_id='district_bank', - bash_command='python /opt/airflow/dags/repo/dags/poi/business/district_bank.py', + task_id='district_facilities', + bash_command='python /opt/airflow/dags/repo/dags/poi/business/district_facilities.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', ) t1 diff --git a/dags/poi/district_hospital_dag.py b/dags/poi/district_hospital_dag.py deleted file mode 100644 index 76ccac3..0000000 --- a/dags/poi/district_hospital_dag.py +++ /dev/null @@ -1,22 +0,0 @@ -""" -小区周边银行DAG -""" -from airflow.operators.bash import BashOperator - -import pendulum - -from airflow import DAG - -with DAG( - "district_hospital_dag", - # 每月1日17点运行 - schedule_interval="0 17 1 * *", - start_date=pendulum.datetime(2022, 6, 20, tz="Asia/Shanghai"), - catchup=True, -) as dag: - t1 = BashOperator( - task_id='district_hospital', - bash_command='python /opt/airflow/dags/repo/dags/poi/business/district_hospital.py', - ) - t1 - diff --git a/dags/poi/district_school_dag.py b/dags/poi/district_school_dag.py deleted file mode 100644 index b4c6eef..0000000 --- a/dags/poi/district_school_dag.py +++ /dev/null @@ -1,22 +0,0 @@ -""" -小区周边银行DAG -""" -from airflow.operators.bash import BashOperator - -import pendulum - -from airflow import DAG - -with DAG( - "district_school_dag", - # 每月2日9点运行 - schedule_interval="0 9 2 * *", - start_date=pendulum.datetime(2022, 6, 20, tz="Asia/Shanghai"), - catchup=True, -) as dag: - t1 = BashOperator( - task_id='district_school', - bash_command='python /opt/airflow/dags/repo/dags/poi/business/district_school.py', - ) - t1 - diff --git a/dags/poi/district_supermarket_dag.py b/dags/poi/district_supermarket_dag.py deleted file mode 100644 index 4269f2a..0000000 --- a/dags/poi/district_supermarket_dag.py +++ /dev/null @@ -1,22 +0,0 @@ -""" -小区周边公交DAG -""" -from airflow.operators.bash import BashOperator - -import pendulum - -from airflow import DAG - -with DAG( - "district_supermarket_dag", - # 每月2日1点运行 - schedule_interval="0 1 2 * *", - start_date=pendulum.datetime(2022, 6, 18, tz="Asia/Shanghai"), - catchup=True, -) as dag: - t1 = BashOperator( - task_id='district_supermarket', - bash_command='python /opt/airflow/dags/repo/dags/poi/business/district_supermarket.py', - ) - t1 -