76 lines
2.4 KiB
Python
76 lines
2.4 KiB
Python
"""
|
|
Example DAG demonstrating the usage of the TaskFlow API to execute Python functions natively and within a
|
|
virtual environment.
|
|
"""
|
|
import logging
|
|
import shutil
|
|
import time
|
|
from datetime import datetime
|
|
from pprint import pprint
|
|
|
|
from airflow import DAG
|
|
from airflow.decorators import task
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
with DAG(
|
|
dag_id='example_python_operator',
|
|
schedule_interval=None,
|
|
start_date=datetime(2021, 1, 1),
|
|
catchup=False,
|
|
tags=['example'],
|
|
) as dag:
|
|
# [START howto_operator_python]
|
|
@task(task_id="print_the_context")
|
|
def print_context(ds=None, **kwargs):
|
|
"""Print the Airflow context and ds variable from the context."""
|
|
pprint(kwargs)
|
|
print(ds)
|
|
return 'Whatever you return gets printed in the logs'
|
|
|
|
run_this = print_context()
|
|
# [END howto_operator_python]
|
|
|
|
# [START howto_operator_python_kwargs]
|
|
# Generate 5 sleeping tasks, sleeping from 0.0 to 0.4 seconds respectively
|
|
for i in range(5):
|
|
|
|
@task(task_id=f'sleep_for_{i}')
|
|
def my_sleeping_function(random_base):
|
|
"""This is a function that will run within the DAG execution"""
|
|
time.sleep(random_base)
|
|
|
|
sleeping_task = my_sleeping_function(random_base=float(i) / 10)
|
|
|
|
run_this >> sleeping_task
|
|
# [END howto_operator_python_kwargs]
|
|
|
|
if not shutil.which("virtualenv"):
|
|
log.warning("The virtalenv_python example task requires virtualenv, please install it.")
|
|
else:
|
|
# [START howto_operator_python_venv]
|
|
@task.virtualenv(
|
|
task_id="virtualenv_python", requirements=["colorama==0.4.0"], system_site_packages=False
|
|
)
|
|
def callable_virtualenv():
|
|
"""
|
|
Example function that will be performed in a virtual environment.
|
|
|
|
Importing at the module level ensures that it will not attempt to import the
|
|
library before it is installed.
|
|
"""
|
|
from time import sleep
|
|
|
|
from colorama import Back, Fore, Style
|
|
|
|
print(Fore.RED + 'some red text')
|
|
print(Back.GREEN + 'and with a green background')
|
|
print(Style.DIM + 'and in dim text')
|
|
print(Style.RESET_ALL)
|
|
for _ in range(10):
|
|
print(Style.DIM + 'Please wait...', flush=True)
|
|
sleep(10)
|
|
print('Finished')
|
|
|
|
virtualenv_task = callable_virtualenv()
|
|
# [END howto_operator_python_venv] |