优化动态安装
This commit is contained in:
+33
-21
@@ -1,26 +1,38 @@
|
||||
from kubernetes.client import models as k8s
|
||||
from airflow import DAG
|
||||
from airflow.providers.cncf.kubernetes.operators.kubernetes_pod import KubernetesPodOperator
|
||||
import pendulum
|
||||
from airflow.operators.python_operator import PythonOperator
|
||||
from airflow.contrib.operators.kubernetes_pod_operator import KubernetesPodOperator
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
init_container = {
|
||||
'name': 'init-container',
|
||||
'image': 'apache/airflow:2.5.1',
|
||||
'command': [
|
||||
'sh',
|
||||
'-c',
|
||||
'pip install -I -r requirements.txt -i https://repo.fangjia.com/repository/pypi-public/simple/'
|
||||
]
|
||||
default_args = {
|
||||
'owner': 'airflow',
|
||||
'start_date': datetime(2023, 3, 10),
|
||||
'retries': 1,
|
||||
'retry_delay': timedelta(minutes=5),
|
||||
}
|
||||
|
||||
with DAG(
|
||||
"install_dag",
|
||||
schedule_interval="0 1 28 * *",
|
||||
start_date=pendulum.datetime(2023, 3, 11, tz="Asia/Shanghai"),
|
||||
catchup=True,
|
||||
description="动态安装依赖"
|
||||
) as dag:
|
||||
task = KubernetesPodOperator(
|
||||
task_id='install',
|
||||
image="apache/airflow:2.5.1",
|
||||
init_containers=[init_container]
|
||||
dag = DAG('install_python_deps', default_args=default_args, schedule_interval=timedelta(days=1))
|
||||
|
||||
def install_python_deps():
|
||||
# Use pip to install dependencies
|
||||
pass
|
||||
|
||||
with dag:
|
||||
t1 = KubernetesPodOperator(
|
||||
task_id='install-python-deps',
|
||||
name='install-python-deps',
|
||||
namespace='default',
|
||||
image='python:3.8',
|
||||
cmds=['bash', '-cx'],
|
||||
arguments=['pip install -r requirements.txt -i https://repo.fangjia.com/repository/pypi-public/simple/'],
|
||||
labels={"app": "airflow"},
|
||||
volumes=[],
|
||||
volume_mounts=[],
|
||||
resources=k8s.V1ResourceRequirements(
|
||||
requests={"cpu": "100m", "memory": "128Mi"},
|
||||
limits={"cpu": "100m", "memory": "128Mi"}),
|
||||
is_delete_operator_pod=True,
|
||||
get_logs=True
|
||||
)
|
||||
|
||||
t1
|
||||
|
||||
Reference in New Issue
Block a user