From b5a2dccaee3e1807fff13715bced7f453bc11b1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=96=87=E8=89=B3?= Date: Thu, 30 Dec 2021 14:28:59 +0800 Subject: [PATCH] Init --- {goojia => dags}/__init__.py | 0 dags/example_python_operator.py | 76 +++++++++++++++++++++++++++++++ {goojia/dags => dags}/tutorial.py | 0 goojia/dags/__init__.py | 0 requirements.txt | 1 + 5 files changed, 77 insertions(+) rename {goojia => dags}/__init__.py (100%) create mode 100644 dags/example_python_operator.py rename {goojia/dags => dags}/tutorial.py (100%) delete mode 100644 goojia/dags/__init__.py create mode 100644 requirements.txt diff --git a/goojia/__init__.py b/dags/__init__.py similarity index 100% rename from goojia/__init__.py rename to dags/__init__.py diff --git a/dags/example_python_operator.py b/dags/example_python_operator.py new file mode 100644 index 0000000..a37f4bb --- /dev/null +++ b/dags/example_python_operator.py @@ -0,0 +1,76 @@ +""" +Example DAG demonstrating the usage of the TaskFlow API to execute Python functions natively and within a +virtual environment. +""" +import logging +import shutil +import time +from datetime import datetime +from pprint import pprint + +from airflow import DAG +from airflow.decorators import task + +log = logging.getLogger(__name__) + +with DAG( + dag_id='example_python_operator', + schedule_interval=None, + start_date=datetime(2021, 1, 1), + catchup=False, + tags=['example'], +) as dag: + # [START howto_operator_python] + @task(task_id="print_the_context") + def print_context(ds=None, **kwargs): + """Print the Airflow context and ds variable from the context.""" + pprint(kwargs) + print(ds) + return 'Whatever you return gets printed in the logs' + + run_this = print_context() + # [END howto_operator_python] + + # [START howto_operator_python_kwargs] + # Generate 5 sleeping tasks, sleeping from 0.0 to 0.4 seconds respectively + for i in range(5): + + @task(task_id=f'sleep_for_{i}') + def my_sleeping_function(random_base): + """This is a function that will run within the DAG execution""" + time.sleep(random_base) + + sleeping_task = my_sleeping_function(random_base=float(i) / 10) + + run_this >> sleeping_task + # [END howto_operator_python_kwargs] + + if not shutil.which("virtualenv"): + log.warning("The virtalenv_python example task requires virtualenv, please install it.") + else: + # [START howto_operator_python_venv] + @task.virtualenv( + task_id="virtualenv_python", requirements=["colorama==0.4.0"], system_site_packages=False + ) + def callable_virtualenv(): + """ + Example function that will be performed in a virtual environment. + + Importing at the module level ensures that it will not attempt to import the + library before it is installed. + """ + from time import sleep + + from colorama import Back, Fore, Style + + print(Fore.RED + 'some red text') + print(Back.GREEN + 'and with a green background') + print(Style.DIM + 'and in dim text') + print(Style.RESET_ALL) + for _ in range(10): + print(Style.DIM + 'Please wait...', flush=True) + sleep(10) + print('Finished') + + virtualenv_task = callable_virtualenv() + # [END howto_operator_python_venv] \ No newline at end of file diff --git a/goojia/dags/tutorial.py b/dags/tutorial.py similarity index 100% rename from goojia/dags/tutorial.py rename to dags/tutorial.py diff --git a/goojia/dags/__init__.py b/goojia/dags/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..067fa52 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +apache-airflow \ No newline at end of file