diff --git a/dags/example_python_operator.py b/dags/example_python_operator.py new file mode 100644 index 0000000..7d155f9 --- /dev/null +++ b/dags/example_python_operator.py @@ -0,0 +1,72 @@ +mport logging +import shutil +import time +from datetime import datetime +from pprint import pprint + +from airflow import DAG +from airflow.decorators import task + +log = logging.getLogger(__name__) + +with DAG( + dag_id='example_python_operator', + schedule_interval=None, + start_date=datetime(2021, 1, 1), + catchup=False, + tags=['example'], +) as dag: + # [START howto_operator_python] + @task(task_id="print_the_context") + def print_context(ds=None, **kwargs): + """Print the Airflow context and ds variable from the context.""" + pprint(kwargs) + print(ds) + return 'Whatever you return gets printed in the logs' + + run_this = print_context() + # [END howto_operator_python] + + # [START howto_operator_python_kwargs] + # Generate 5 sleeping tasks, sleeping from 0.0 to 0.4 seconds respectively + for i in range(5): + + @task(task_id=f'sleep_for_{i}') + def my_sleeping_function(random_base): + """This is a function that will run within the DAG execution""" + time.sleep(random_base) + + sleeping_task = my_sleeping_function(random_base=float(i) / 10) + + run_this >> sleeping_task + # [END howto_operator_python_kwargs] + + if not shutil.which("virtualenv"): + log.warning("The virtalenv_python example task requires virtualenv, please install it.") + else: + # [START howto_operator_python_venv] + @task.virtualenv( + task_id="virtualenv_python", requirements=["colorama==0.4.0"], system_site_packages=False + ) + def callable_virtualenv(): + """ + Example function that will be performed in a virtual environment. + + Importing at the module level ensures that it will not attempt to import the + library before it is installed. + """ + from time import sleep + + from colorama import Back, Fore, Style + + print(Fore.RED + 'some red text') + print(Back.GREEN + 'and with a green background') + print(Style.DIM + 'and in dim text') + print(Style.RESET_ALL) + for _ in range(10): + print(Style.DIM + 'Please wait...', flush=True) + sleep(10) + print('Finished') + + virtualenv_task = callable_virtualenv() + # [END howto_operator_python_venv] \ No newline at end of file