From c8093fce87e6847a0ead7f5d62abbe599a99cf09 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Sat, 11 Mar 2023 17:27:16 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=8A=A8=E6=80=81=E5=AE=89?= =?UTF-8?q?=E8=A3=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/business/amap_key_generate_dag.py | 2 +- dags/install_dag.py | 40 -------------------------- dags/parent_dag.py | 21 ++++++++++++++ 3 files changed, 22 insertions(+), 41 deletions(-) delete mode 100644 dags/install_dag.py create mode 100644 dags/parent_dag.py diff --git a/dags/business/amap_key_generate_dag.py b/dags/business/amap_key_generate_dag.py index 31fa783..92a5c0c 100644 --- a/dags/business/amap_key_generate_dag.py +++ b/dags/business/amap_key_generate_dag.py @@ -1,6 +1,6 @@ from airflow.operators.bash import BashOperator import pendulum -from airflow import DAG +from ..parent_dag import DAG with DAG( "amap_key_generate_dag", diff --git a/dags/install_dag.py b/dags/install_dag.py deleted file mode 100644 index fed3e55..0000000 --- a/dags/install_dag.py +++ /dev/null @@ -1,40 +0,0 @@ -from datetime import datetime, timedelta -from airflow import DAG -from airflow.contrib.operators.kubernetes_pod_operator import KubernetesPodOperator - -dag = DAG( - dag_id='install_dag', - start_date=datetime(2023, 3, 11), - schedule_interval=None, - catchup=False -) - -task1 = KubernetesPodOperator( - task_id='task1', - name='task1', - # 注意修改命名空间,否则无权限 - namespace='airflow-plus', - image='apache/airflow:2.5.1', - cmds=['bash', '-cx'], - # 安装依赖项目 - arguments=[ - 'pip install elasticsearch==7.11.0 transbigdata==0.4.15 pykalman==0.9.5 -i https://repo.fangjia.com/repository/pypi-public/simple/', - 'pip install -I goojia_utils -i https://repo.fangjia.com/repository/pypi-public/simple/'], - labels={'app': 'my-app', 'component': 'my-component'}, - affinity={ - 'podAffinity': { - 'requiredDuringSchedulingIgnoredDuringExecution': [{ - 'labelSelector': { - 'matchExpressions': [{ - 'key': 'component', - 'operator': 'In', - 'values': ['worker'] - }] - }, - 'topologyKey': 'kubernetes.io/hostname' - }] - } - }, - dag=dag -) - diff --git a/dags/parent_dag.py b/dags/parent_dag.py new file mode 100644 index 0000000..08d3e75 --- /dev/null +++ b/dags/parent_dag.py @@ -0,0 +1,21 @@ +from airflow import DAG as AirflowDAG +from airflow.models import DagRun +from airflow.utils.dates import cron_presets, parse_execution_date +from airflow.configuration import conf +import subprocess +import sys + +class DAG(AirflowDAG): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + if conf.getboolean('core', 'security') and 'install_dependencies' not in self.dag_id: + self._install_dependencies() + + def _install_dependencies(self): + subprocess.check_call([sys.executable, "-m", "pip install elasticsearch==7.11.0 transbigdata==0.4.15 pykalman==0.9.5 -i https://repo.fangjia.com/repository/pypi-public/simple/", + "pip install -I goojia_utils -i https://repo.fangjia.com/repository/pypi-public/simple/"]) + + def run(self, *args, **kwargs): + self._install_dependencies() + super().run(*args, **kwargs) \ No newline at end of file