优化动态安装
This commit is contained in:
@@ -1,32 +0,0 @@
|
||||
from kubernetes.client import models as k8s
|
||||
from airflow import DAG
|
||||
from airflow.operators.python_operator import PythonOperator
|
||||
from airflow.contrib.operators.kubernetes_pod_operator import KubernetesPodOperator
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
default_args = {
|
||||
'owner': 'airflow',
|
||||
'start_date': datetime(2023, 3, 10),
|
||||
'retry_delay': timedelta(minutes=5),
|
||||
}
|
||||
|
||||
dag = DAG('install_dag', default_args=default_args, schedule_interval=None)
|
||||
|
||||
with dag:
|
||||
t1 = KubernetesPodOperator(
|
||||
task_id='install',
|
||||
name='install',
|
||||
# 注意修改命名空间,否则无权限
|
||||
namespace='airflow-plus',
|
||||
image='python:3.8',
|
||||
cmds=['bash', '-cx'],
|
||||
# 安装依赖项目
|
||||
arguments=['pip install elasticsearch==7.11.0 transbigdata==0.4.15 pykalman==0.9.5 -i https://repo.fangjia.com/repository/pypi-public/simple/',
|
||||
'pip install -I goojia_utils -i https://repo.fangjia.com/repository/pypi-public/simple/'],
|
||||
labels={"release": "airflow"},
|
||||
volumes=[],
|
||||
volume_mounts=[],
|
||||
get_logs=True
|
||||
)
|
||||
|
||||
t1
|
||||
@@ -0,0 +1,40 @@
|
||||
from datetime import datetime, timedelta
|
||||
from airflow import DAG
|
||||
from airflow.contrib.operators.kubernetes_pod_operator import KubernetesPodOperator
|
||||
|
||||
dag = DAG(
|
||||
dag_id='my_dag',
|
||||
start_date=datetime(2023, 3, 11),
|
||||
schedule_interval=None,
|
||||
catchup=False
|
||||
)
|
||||
|
||||
task1 = KubernetesPodOperator(
|
||||
task_id='task1',
|
||||
name='task1',
|
||||
# 注意修改命名空间,否则无权限
|
||||
namespace='airflow-plus',
|
||||
image='apache/airflow:2.5.1',
|
||||
cmds=['bash', '-cx'],
|
||||
# 安装依赖项目
|
||||
arguments=[
|
||||
'pip install elasticsearch==7.11.0 transbigdata==0.4.15 pykalman==0.9.5 -i https://repo.fangjia.com/repository/pypi-public/simple/',
|
||||
'pip install -I goojia_utils -i https://repo.fangjia.com/repository/pypi-public/simple/'],
|
||||
labels={'app': 'my-app', 'component': 'my-component'},
|
||||
affinity={
|
||||
'podAffinity': {
|
||||
'requiredDuringSchedulingIgnoredDuringExecution': [{
|
||||
'labelSelector': {
|
||||
'matchExpressions': [{
|
||||
'key': 'component',
|
||||
'operator': 'In',
|
||||
'values': ['worker']
|
||||
}]
|
||||
},
|
||||
'topologyKey': 'kubernetes.io/hostname'
|
||||
}]
|
||||
}
|
||||
},
|
||||
dag=dag
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user