From cbd3f8dfd2e65e265c7d239d2ac560119a11ef7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=96=87=E8=89=B3?= Date: Wed, 29 Dec 2021 19:13:03 +0800 Subject: [PATCH] Init --- .gitignore | 2 ++ dags/tutorial.py | 87 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+) create mode 100644 .gitignore create mode 100644 dags/tutorial.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d2c55d6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/venv/ +.idea diff --git a/dags/tutorial.py b/dags/tutorial.py new file mode 100644 index 0000000..b3cb38e --- /dev/null +++ b/dags/tutorial.py @@ -0,0 +1,87 @@ +from datetime import datetime, timedelta +from textwrap import dedent + +# The DAG object; we'll need this to instantiate a DAG +from airflow import DAG + +# Operators; we need this to operate! +from airflow.operators.bash import BashOperator +# These args will get passed on to each operator +# You can override them on a per-task basis during operator initialization +default_args = { + 'owner': 'airflow', + 'depends_on_past': False, + 'email': ['airflow@example.com'], + 'email_on_failure': False, + 'email_on_retry': False, + 'retries': 1, + 'retry_delay': timedelta(minutes=5), + # 'queue': 'bash_queue', + # 'pool': 'backfill', + # 'priority_weight': 10, + # 'end_date': datetime(2016, 1, 1), + # 'wait_for_downstream': False, + # 'dag': dag, + # 'sla': timedelta(hours=2), + # 'execution_timeout': timedelta(seconds=300), + # 'on_failure_callback': some_function, + # 'on_success_callback': some_other_function, + # 'on_retry_callback': another_function, + # 'sla_miss_callback': yet_another_function, + # 'trigger_rule': 'all_success' +} +with DAG( + 'tutorial', + default_args=default_args, + description='A simple tutorial DAG', + schedule_interval=timedelta(days=1), + start_date=datetime(2021, 1, 1), + catchup=False, + tags=['example'], +) as dag: + + # t1, t2 and t3 are examples of tasks created by instantiating operators + t1 = BashOperator( + task_id='print_date', + bash_command='date', + ) + + t2 = BashOperator( + task_id='sleep', + depends_on_past=False, + bash_command='sleep 5', + retries=3, + ) + t1.doc_md = dedent( + """\ + #### Task Documentation + You can document your task using the attributes `doc_md` (markdown), + `doc` (plain text), `doc_rst`, `doc_json`, `doc_yaml` which gets + rendered in the UI's Task Instance Details page. + ![img](http://montcs.bloomu.edu/~bobmon/Semesters/2012-01/491/import%20soul.png) + + """ + ) + + dag.doc_md = __doc__ # providing that you have a docstring at the beginning of the DAG + dag.doc_md = """ + This is a documentation placed anywhere + """ # otherwise, type it like this + templated_command = dedent( + """ + {% for i in range(5) %} + echo "{{ ds }}" + echo "{{ macros.ds_add(ds, 7)}}" + echo "{{ params.my_param }}" + {% endfor %} + """ + ) + + t3 = BashOperator( + task_id='templated', + depends_on_past=False, + bash_command=templated_command, + params={'my_param': 'Parameter I passed in'}, + ) + + t1 >> [t2, t3] \ No newline at end of file