Files
airflow/dags/init_dag.py
T
2023-03-11 13:29:19 +08:00

39 lines
1.2 KiB
Python

from kubernetes.client import models as k8s
from airflow import DAG
from airflow.operators.python_operator import PythonOperator
from airflow.contrib.operators.kubernetes_pod_operator import KubernetesPodOperator
from datetime import datetime, timedelta
default_args = {
'owner': 'airflow',
'start_date': datetime(2023, 3, 10),
'retries': 1,
'retry_delay': timedelta(minutes=5),
}
dag = DAG('install_python_deps', default_args=default_args, schedule_interval=timedelta(days=1))
def install_python_deps():
# Use pip to install dependencies
pass
with dag:
t1 = KubernetesPodOperator(
task_id='install-python-deps',
name='install-python-deps',
namespace='default',
image='python:3.8',
cmds=['bash', '-cx'],
arguments=['pip install -r requirements.txt -i https://repo.fangjia.com/repository/pypi-public/simple/'],
labels={"app": "airflow"},
volumes=[],
volume_mounts=[],
resources=k8s.V1ResourceRequirements(
requests={"cpu": "100m", "memory": "128Mi"},
limits={"cpu": "100m", "memory": "128Mi"}),
is_delete_operator_pod=True,
get_logs=True
)
t1