优化动态安装

This commit is contained in:
Jian Shen
2023-03-11 17:27:16 +08:00
parent e581409541
commit c8093fce87
3 changed files with 22 additions and 41 deletions
+1 -1
View File
@@ -1,6 +1,6 @@
from airflow.operators.bash import BashOperator
import pendulum
from airflow import DAG
from ..parent_dag import DAG
with DAG(
"amap_key_generate_dag",
-40
View File
@@ -1,40 +0,0 @@
from datetime import datetime, timedelta
from airflow import DAG
from airflow.contrib.operators.kubernetes_pod_operator import KubernetesPodOperator
dag = DAG(
dag_id='install_dag',
start_date=datetime(2023, 3, 11),
schedule_interval=None,
catchup=False
)
task1 = KubernetesPodOperator(
task_id='task1',
name='task1',
# 注意修改命名空间,否则无权限
namespace='airflow-plus',
image='apache/airflow:2.5.1',
cmds=['bash', '-cx'],
# 安装依赖项目
arguments=[
'pip install elasticsearch==7.11.0 transbigdata==0.4.15 pykalman==0.9.5 -i https://repo.fangjia.com/repository/pypi-public/simple/',
'pip install -I goojia_utils -i https://repo.fangjia.com/repository/pypi-public/simple/'],
labels={'app': 'my-app', 'component': 'my-component'},
affinity={
'podAffinity': {
'requiredDuringSchedulingIgnoredDuringExecution': [{
'labelSelector': {
'matchExpressions': [{
'key': 'component',
'operator': 'In',
'values': ['worker']
}]
},
'topologyKey': 'kubernetes.io/hostname'
}]
}
},
dag=dag
)
+21
View File
@@ -0,0 +1,21 @@
from airflow import DAG as AirflowDAG
from airflow.models import DagRun
from airflow.utils.dates import cron_presets, parse_execution_date
from airflow.configuration import conf
import subprocess
import sys
class DAG(AirflowDAG):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
if conf.getboolean('core', 'security') and 'install_dependencies' not in self.dag_id:
self._install_dependencies()
def _install_dependencies(self):
subprocess.check_call([sys.executable, "-m", "pip install elasticsearch==7.11.0 transbigdata==0.4.15 pykalman==0.9.5 -i https://repo.fangjia.com/repository/pypi-public/simple/",
"pip install -I goojia_utils -i https://repo.fangjia.com/repository/pypi-public/simple/"])
def run(self, *args, **kwargs):
self._install_dependencies()
super().run(*args, **kwargs)