From b747587402eef2bb8c115d05f7ab520268b61095 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Sat, 11 Mar 2023 16:41:19 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=8A=A8=E6=80=81=E5=AE=89?= =?UTF-8?q?=E8=A3=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/init_dag.py | 32 -------------------------------- dags/install_dag.py | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 32 deletions(-) delete mode 100644 dags/init_dag.py create mode 100644 dags/install_dag.py diff --git a/dags/init_dag.py b/dags/init_dag.py deleted file mode 100644 index 3d8bab1..0000000 --- a/dags/init_dag.py +++ /dev/null @@ -1,32 +0,0 @@ -from kubernetes.client import models as k8s -from airflow import DAG -from airflow.operators.python_operator import PythonOperator -from airflow.contrib.operators.kubernetes_pod_operator import KubernetesPodOperator -from datetime import datetime, timedelta - -default_args = { - 'owner': 'airflow', - 'start_date': datetime(2023, 3, 10), - 'retry_delay': timedelta(minutes=5), -} - -dag = DAG('install_dag', default_args=default_args, schedule_interval=None) - -with dag: - t1 = KubernetesPodOperator( - task_id='install', - name='install', - # 注意修改命名空间,否则无权限 - namespace='airflow-plus', - image='python:3.8', - cmds=['bash', '-cx'], - # 安装依赖项目 - arguments=['pip install elasticsearch==7.11.0 transbigdata==0.4.15 pykalman==0.9.5 -i https://repo.fangjia.com/repository/pypi-public/simple/', - 'pip install -I goojia_utils -i https://repo.fangjia.com/repository/pypi-public/simple/'], - labels={"release": "airflow"}, - volumes=[], - volume_mounts=[], - get_logs=True - ) - - t1 diff --git a/dags/install_dag.py b/dags/install_dag.py new file mode 100644 index 0000000..dc9426c --- /dev/null +++ b/dags/install_dag.py @@ -0,0 +1,40 @@ +from datetime import datetime, timedelta +from airflow import DAG +from airflow.contrib.operators.kubernetes_pod_operator import KubernetesPodOperator + +dag = DAG( + dag_id='my_dag', + start_date=datetime(2023, 3, 11), + schedule_interval=None, + catchup=False +) + +task1 = KubernetesPodOperator( + task_id='task1', + name='task1', + # 注意修改命名空间,否则无权限 + namespace='airflow-plus', + image='apache/airflow:2.5.1', + cmds=['bash', '-cx'], + # 安装依赖项目 + arguments=[ + 'pip install elasticsearch==7.11.0 transbigdata==0.4.15 pykalman==0.9.5 -i https://repo.fangjia.com/repository/pypi-public/simple/', + 'pip install -I goojia_utils -i https://repo.fangjia.com/repository/pypi-public/simple/'], + labels={'app': 'my-app', 'component': 'my-component'}, + affinity={ + 'podAffinity': { + 'requiredDuringSchedulingIgnoredDuringExecution': [{ + 'labelSelector': { + 'matchExpressions': [{ + 'key': 'component', + 'operator': 'In', + 'values': ['worker'] + }] + }, + 'topologyKey': 'kubernetes.io/hostname' + }] + } + }, + dag=dag +) +