优化动态安装

This commit is contained in:
Jian Shen
2023-03-11 13:29:19 +08:00
parent 568a06fe0b
commit e065f8632b
+30 -21
View File
@@ -1,26 +1,35 @@
from kubernetes.client import models as k8s
from airflow import DAG
from airflow.providers.cncf.kubernetes.operators.kubernetes_pod import KubernetesPodOperator
import pendulum
from airflow.operators.python_operator import PythonOperator
from airflow.contrib.operators.kubernetes_pod_operator import KubernetesPodOperator
from datetime import datetime, timedelta
init_container = {
'name': 'init-container',
'image': 'apache/airflow:2.5.1',
'command': [
'sh',
'-c',
'pip install -I -r requirements.txt -i https://repo.fangjia.com/repository/pypi-public/simple/'
]
default_args = {
'owner': 'airflow',
'start_date': datetime(2023, 3, 10),
'retries': 1,
'retry_delay': timedelta(minutes=5),
}
with DAG(
"install_dag",
schedule_interval="0 1 28 * *",
start_date=pendulum.datetime(2023, 3, 11, tz="Asia/Shanghai"),
catchup=True,
description="动态安装依赖"
) as dag:
task = KubernetesPodOperator(
task_id='install',
image="apache/airflow:2.5.1",
init_containers=[init_container]
dag = DAG('install_python_deps', default_args=default_args, schedule_interval=timedelta(days=1))
def install_python_deps():
# Use pip to install dependencies
pass
with dag:
t1 = KubernetesPodOperator(
task_id='install-python-deps',
name='install-python-deps',
namespace='default',
image='python:3.8',
cmds=['bash', '-cx'],
arguments=['pip install -r requirements.txt -i https://repo.fangjia.com/repository/pypi-public/simple/'],
labels={"app": "airflow"},
volumes=[],
volume_mounts=[],
is_delete_operator_pod=True,
get_logs=True
)
t1