安装依赖放置在第一个任务中,便于日志查看

This commit is contained in:
shenjian@fangjia.com
2023-03-14 13:36:13 +08:00
parent c0fcafc2fc
commit 78f210db5d
7 changed files with 13 additions and 34 deletions
-2
View File
@@ -1,2 +0,0 @@
import sys
sys.path.insert(0, '/opt/airflow/dags/repo')
+2 -5
View File
@@ -3,18 +3,15 @@ sys.path.insert(0, '/opt/airflow/dags/repo')
from airflow.operators.bash import BashOperator
import pendulum
from dags.my_dag import MyDAG
import logging
log = logging.getLogger(__name__)
from airflow import DAG
with MyDAG(
with DAG(
"amap_key_generate_dag",
# 每天运行
schedule_interval="0 2 * * *",
start_date=pendulum.datetime(2023, 3, 11, tz="Asia/Shanghai"),
catchup=True,
) as dag:
log.info("enter in MyDAG CHILD")
t1 = BashOperator(
task_id='amap_key_generate',
bash_command='python /opt/airflow/dags/repo/dags/business/business/amap_key_generate.py',
@@ -1,6 +1,9 @@
from goojia_utils import REDIS_CLIENT
import json
import dags.install as install
install()
"""
高德key每日定时生成
"""
+8
View File
@@ -0,0 +1,8 @@
import logging
import os
log = logging.getLogger(__name__)
log.info("begin install requirements")
os.system("pip install -I -r /opt/airflow/dags/repo/dags/requirements.txt -i https://repo.fangjia.com/repository/pypi-public/simple/")
log.info("finish install requirements")
-26
View File
@@ -1,26 +0,0 @@
from datetime import datetime
from airflow.operators.bash import BashOperator
from airflow.models import DAG
import logging
log = logging.getLogger(__name__)
class MyDAG(DAG):
pass
class MyDAG(DAG):
log.info("enter in MyDAG")
def __init__(self, *args, **kwargs):
log.info('SUCCESS')
super().__init__(
*args, **kwargs
)
install = BashOperator(
task_id='install_required',
bash_command='pip install -I -r requirements.txt -i https://repo.fangjia.com/repository/pypi-public/simple/',
)
install
log.info('OK')
-1
View File
@@ -1 +0,0 @@
pip install -I -r requirements.txt -i https://repo.fangjia.com/repository/pypi-public/simple/