Init
This commit is contained in:
@@ -0,0 +1,76 @@
|
||||
"""
|
||||
Example DAG demonstrating the usage of the TaskFlow API to execute Python functions natively and within a
|
||||
virtual environment.
|
||||
"""
|
||||
import logging
|
||||
import shutil
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pprint import pprint
|
||||
|
||||
from airflow import DAG
|
||||
from airflow.decorators import task
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
with DAG(
|
||||
dag_id='example_python_operator',
|
||||
schedule_interval=None,
|
||||
start_date=datetime(2021, 1, 1),
|
||||
catchup=False,
|
||||
tags=['example'],
|
||||
) as dag:
|
||||
# [START howto_operator_python]
|
||||
@task(task_id="print_the_context")
|
||||
def print_context(ds=None, **kwargs):
|
||||
"""Print the Airflow context and ds variable from the context."""
|
||||
pprint(kwargs)
|
||||
print(ds)
|
||||
return 'Whatever you return gets printed in the logs'
|
||||
|
||||
run_this = print_context()
|
||||
# [END howto_operator_python]
|
||||
|
||||
# [START howto_operator_python_kwargs]
|
||||
# Generate 5 sleeping tasks, sleeping from 0.0 to 0.4 seconds respectively
|
||||
for i in range(5):
|
||||
|
||||
@task(task_id=f'sleep_for_{i}')
|
||||
def my_sleeping_function(random_base):
|
||||
"""This is a function that will run within the DAG execution"""
|
||||
time.sleep(random_base)
|
||||
|
||||
sleeping_task = my_sleeping_function(random_base=float(i) / 10)
|
||||
|
||||
run_this >> sleeping_task
|
||||
# [END howto_operator_python_kwargs]
|
||||
|
||||
if not shutil.which("virtualenv"):
|
||||
log.warning("The virtalenv_python example task requires virtualenv, please install it.")
|
||||
else:
|
||||
# [START howto_operator_python_venv]
|
||||
@task.virtualenv(
|
||||
task_id="virtualenv_python", requirements=["colorama==0.4.0"], system_site_packages=False
|
||||
)
|
||||
def callable_virtualenv():
|
||||
"""
|
||||
Example function that will be performed in a virtual environment.
|
||||
|
||||
Importing at the module level ensures that it will not attempt to import the
|
||||
library before it is installed.
|
||||
"""
|
||||
from time import sleep
|
||||
|
||||
from colorama import Back, Fore, Style
|
||||
|
||||
print(Fore.RED + 'some red text')
|
||||
print(Back.GREEN + 'and with a green background')
|
||||
print(Style.DIM + 'and in dim text')
|
||||
print(Style.RESET_ALL)
|
||||
for _ in range(10):
|
||||
print(Style.DIM + 'Please wait...', flush=True)
|
||||
sleep(10)
|
||||
print('Finished')
|
||||
|
||||
virtualenv_task = callable_virtualenv()
|
||||
# [END howto_operator_python_venv]
|
||||
@@ -0,0 +1,87 @@
|
||||
from datetime import datetime, timedelta
|
||||
from textwrap import dedent
|
||||
#Test
|
||||
# The DAG object; we'll need this to instantiate a DAG
|
||||
from airflow import DAG
|
||||
import pandas as pd
|
||||
# Operators; we need this to operate!
|
||||
from airflow.operators.bash import BashOperator
|
||||
# These args will get passed on to each operator
|
||||
# You can override them on a per-task basis during operator initialization
|
||||
default_args = {
|
||||
'owner': 'airflow',
|
||||
'depends_on_past': False,
|
||||
'email': ['airflow@example.com'],
|
||||
'email_on_failure': False,
|
||||
'email_on_retry': False,
|
||||
'retries': 1,
|
||||
'retry_delay': timedelta(minutes=5),
|
||||
# 'queue': 'bash_queue',
|
||||
# 'pool': 'backfill',
|
||||
# 'priority_weight': 10,
|
||||
# 'end_date': datetime(2016, 1, 1),
|
||||
# 'wait_for_downstream': False,
|
||||
# 'dag': dag,
|
||||
# 'sla': timedelta(hours=2),
|
||||
# 'execution_timeout': timedelta(seconds=300),
|
||||
# 'on_failure_callback': some_function,
|
||||
# 'on_success_callback': some_other_function,
|
||||
# 'on_retry_callback': another_function,
|
||||
# 'sla_miss_callback': yet_another_function,
|
||||
# 'trigger_rule': 'all_success'
|
||||
}
|
||||
with DAG(
|
||||
'tutorial',
|
||||
default_args=default_args,
|
||||
description='A simple tutorial DAG',
|
||||
schedule_interval=timedelta(days=1),
|
||||
start_date=datetime(2021, 1, 1),
|
||||
catchup=False,
|
||||
tags=['example'],
|
||||
) as dag:
|
||||
|
||||
# t1, t2 and t3 are examples of tasks created by instantiating operators
|
||||
t1 = BashOperator(
|
||||
task_id='print_date',
|
||||
bash_command='date',
|
||||
)
|
||||
|
||||
t2 = BashOperator(
|
||||
task_id='sleep',
|
||||
depends_on_past=False,
|
||||
bash_command='sleep 5',
|
||||
retries=3,
|
||||
)
|
||||
t1.doc_md = dedent(
|
||||
"""\
|
||||
#### Task Documentation
|
||||
You can document your task using the attributes `doc_md` (markdown),
|
||||
`doc` (plain text), `doc_rst`, `doc_json`, `doc_yaml` which gets
|
||||
rendered in the UI's Task Instance Details page.
|
||||

|
||||
|
||||
"""
|
||||
)
|
||||
|
||||
dag.doc_md = __doc__ # providing that you have a docstring at the beginning of the DAG
|
||||
dag.doc_md = """
|
||||
This is a documentation placed anywhere
|
||||
""" # otherwise, type it like this
|
||||
templated_command = dedent(
|
||||
"""
|
||||
{% for i in range(5) %}
|
||||
echo "{{ ds }}"
|
||||
echo "{{ macros.ds_add(ds, 7)}}"
|
||||
echo "{{ params.my_param }}"
|
||||
{% endfor %}
|
||||
"""
|
||||
)
|
||||
|
||||
t3 = BashOperator(
|
||||
task_id='templated',
|
||||
depends_on_past=False,
|
||||
bash_command=templated_command,
|
||||
params={'my_param': 'Parameter I passed in'},
|
||||
)
|
||||
|
||||
t1 >> [t2, t3]
|
||||
Reference in New Issue
Block a user