Files
airflow/dags/business/dag_comm_add.py
T
shenjian@fangjia.com 89e5814468 优化镜像拉取
2023-05-29 15:28:47 +08:00

59 lines
2.6 KiB
Python

from airflow.providers.docker.operators.docker import DockerOperator
from airflow.providers.cncf.kubernetes.operators.kubernetes_pod import KubernetesPodOperator
import pendulum
from airflow import DAG
from docker.types import Mount
from kubernetes.client import models as k8s
with DAG(
dag_id="comm_add_dag",
# 每月20日1点整分运行
schedule_interval="0 1 20 * *",
start_date=pendulum.datetime(2023, 5, 8, tz="Asia/Shanghai"),
catchup=True,
description="小区自动化入库"
) as dag:
# k = KubernetesPodOperator(
# name="comm-add",
# image="192.168.0.19:9012/comm-add:1.0.1",
# cmds=["bash", "-cx"],
# arguments=["echo", "10"],
# labels={"foo": "bar"},
# task_id="comm_add",
# do_xcom_push=True,
# )
# k.dry_run()
production_task = KubernetesPodOperator(namespace='airflow-plus',
# image="my-production-job:release-1.0.1", <-- old release
image="192.168.0.19:9012/comm-add:1.0.1",
image_pull_policy="Always",
image_pull_secrets=[k8s.V1LocalObjectReference("nexus-registry-secret")],
cmds=["python", "main.py"],
arguments=["--city", '{{dag_run.conf["city"] if "city" in dag_run.conf else "all"}}'],
name="comm_add",
task_id="comm_add",
get_logs=True
)
production_task
# t1 = DockerOperator(
# # 刚才定义的conn id
# docker_conn_id="docker_repo",
# # WebServer中点击Admin -> Providers -> Docker对应的Version,如果是auto,可能会报错
# api_version="2.2.0",
# auto_remove=True,
# docker_url='unix://var/run/docker.sock',
# network_mode='bridge',
# image='192.168.0.19:9012/comm-add:1.0.1',
# # 临沂为我们的传参,python代码中, 镜像中部分代码
# # @ click.command()
# # @ click.option('--city', default="all", help='要计算的城市')
# #command='python main.py --city {{dag_run.conf["city"] if "city" in dag_run.conf}} else all',
# command='echo "this is a test message shown from within the container',
# mounts=[
# Mount(source="/var/run/docker.sock", target="/var/run/docker.sock", type="bind")
# ],
# task_id='comm_add'
# )
# t1