From 8345545b0ef931ce56ffbc7564cc218664964dd3 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Sat, 11 Mar 2023 13:29:19 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=8A=A8=E6=80=81=E5=AE=89?= =?UTF-8?q?=E8=A3=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/init_dag.py | 54 +++++++++++++++++++++++++++++------------------- 1 file changed, 33 insertions(+), 21 deletions(-) diff --git a/dags/init_dag.py b/dags/init_dag.py index 5e2df83..bdf4960 100644 --- a/dags/init_dag.py +++ b/dags/init_dag.py @@ -1,26 +1,38 @@ +from kubernetes.client import models as k8s from airflow import DAG -from airflow.providers.cncf.kubernetes.operators.kubernetes_pod import KubernetesPodOperator -import pendulum +from airflow.operators.python_operator import PythonOperator +from airflow.contrib.operators.kubernetes_pod_operator import KubernetesPodOperator +from datetime import datetime, timedelta -init_container = { - 'name': 'init-container', - 'image': 'apache/airflow:2.5.1', - 'command': [ - 'sh', - '-c', - 'pip install -I -r requirements.txt -i https://repo.fangjia.com/repository/pypi-public/simple/' - ] +default_args = { + 'owner': 'airflow', + 'start_date': datetime(2023, 3, 10), + 'retries': 1, + 'retry_delay': timedelta(minutes=5), } -with DAG( - "install_dag", - schedule_interval="0 1 28 * *", - start_date=pendulum.datetime(2023, 3, 11, tz="Asia/Shanghai"), - catchup=True, - description="动态安装依赖" -) as dag: - task = KubernetesPodOperator( - task_id='install', - image="apache/airflow:2.5.1", - init_containers=[init_container] +dag = DAG('install_python_deps', default_args=default_args, schedule_interval=timedelta(days=1)) + +def install_python_deps(): + # Use pip to install dependencies + pass + +with dag: + t1 = KubernetesPodOperator( + task_id='install-python-deps', + name='install-python-deps', + namespace='default', + image='python:3.8', + cmds=['bash', '-cx'], + arguments=['pip install -r requirements.txt -i https://repo.fangjia.com/repository/pypi-public/simple/'], + labels={"app": "airflow"}, + volumes=[], + volume_mounts=[], + resources=k8s.V1ResourceRequirements( + requests={"cpu": "100m", "memory": "128Mi"}, + limits={"cpu": "100m", "memory": "128Mi"}), + is_delete_operator_pod=True, + get_logs=True ) + + t1