diff --git a/dags/example_python_operator.py b/dags/example_python_operator.py deleted file mode 100644 index a37f4bb..0000000 --- a/dags/example_python_operator.py +++ /dev/null @@ -1,76 +0,0 @@ -""" -Example DAG demonstrating the usage of the TaskFlow API to execute Python functions natively and within a -virtual environment. -""" -import logging -import shutil -import time -from datetime import datetime -from pprint import pprint - -from airflow import DAG -from airflow.decorators import task - -log = logging.getLogger(__name__) - -with DAG( - dag_id='example_python_operator', - schedule_interval=None, - start_date=datetime(2021, 1, 1), - catchup=False, - tags=['example'], -) as dag: - # [START howto_operator_python] - @task(task_id="print_the_context") - def print_context(ds=None, **kwargs): - """Print the Airflow context and ds variable from the context.""" - pprint(kwargs) - print(ds) - return 'Whatever you return gets printed in the logs' - - run_this = print_context() - # [END howto_operator_python] - - # [START howto_operator_python_kwargs] - # Generate 5 sleeping tasks, sleeping from 0.0 to 0.4 seconds respectively - for i in range(5): - - @task(task_id=f'sleep_for_{i}') - def my_sleeping_function(random_base): - """This is a function that will run within the DAG execution""" - time.sleep(random_base) - - sleeping_task = my_sleeping_function(random_base=float(i) / 10) - - run_this >> sleeping_task - # [END howto_operator_python_kwargs] - - if not shutil.which("virtualenv"): - log.warning("The virtalenv_python example task requires virtualenv, please install it.") - else: - # [START howto_operator_python_venv] - @task.virtualenv( - task_id="virtualenv_python", requirements=["colorama==0.4.0"], system_site_packages=False - ) - def callable_virtualenv(): - """ - Example function that will be performed in a virtual environment. - - Importing at the module level ensures that it will not attempt to import the - library before it is installed. - """ - from time import sleep - - from colorama import Back, Fore, Style - - print(Fore.RED + 'some red text') - print(Back.GREEN + 'and with a green background') - print(Style.DIM + 'and in dim text') - print(Style.RESET_ALL) - for _ in range(10): - print(Style.DIM + 'Please wait...', flush=True) - sleep(10) - print('Finished') - - virtualenv_task = callable_virtualenv() - # [END howto_operator_python_venv] \ No newline at end of file