59 lines
2.6 KiB
Python
59 lines
2.6 KiB
Python
from airflow.providers.docker.operators.docker import DockerOperator
|
|
from airflow.providers.cncf.kubernetes.operators.kubernetes_pod import KubernetesPodOperator
|
|
import pendulum
|
|
from airflow import DAG
|
|
from docker.types import Mount
|
|
from kubernetes.client import models as k8s
|
|
|
|
with DAG(
|
|
dag_id="comm_add_dag",
|
|
# 每月20日1点整分运行
|
|
schedule_interval="0 1 20 * *",
|
|
start_date=pendulum.datetime(2023, 5, 8, tz="Asia/Shanghai"),
|
|
catchup=True,
|
|
description="小区自动化入库"
|
|
) as dag:
|
|
# k = KubernetesPodOperator(
|
|
# name="comm-add",
|
|
# image="192.168.0.19:9012/comm-add:1.0.1",
|
|
# cmds=["bash", "-cx"],
|
|
# arguments=["echo", "10"],
|
|
# labels={"foo": "bar"},
|
|
# task_id="comm_add",
|
|
# do_xcom_push=True,
|
|
# )
|
|
# k.dry_run()
|
|
|
|
production_task = KubernetesPodOperator(namespace='airflow-plus',
|
|
# image="my-production-job:release-1.0.1", <-- old release
|
|
image="192.168.0.19:9012/comm-add:1.0.1",
|
|
image_pull_policy="Always",
|
|
image_pull_secrets=[k8s.V1LocalObjectReference("nexus-registry-secret")],
|
|
cmds=["python", "main.py"],
|
|
arguments=["--city", '{{dag_run.conf["city"] if "city" in dag_run.conf else "all"}}'],
|
|
name="comm_add",
|
|
task_id="comm_add",
|
|
get_logs=True
|
|
)
|
|
|
|
production_task
|
|
# t1 = DockerOperator(
|
|
# # 刚才定义的conn id
|
|
# docker_conn_id="docker_repo",
|
|
# # WebServer中点击Admin -> Providers -> Docker对应的Version,如果是auto,可能会报错
|
|
# api_version="2.2.0",
|
|
# auto_remove=True,
|
|
# docker_url='unix://var/run/docker.sock',
|
|
# network_mode='bridge',
|
|
# image='192.168.0.19:9012/comm-add:1.0.1',
|
|
# # 临沂为我们的传参,python代码中, 镜像中部分代码
|
|
# # @ click.command()
|
|
# # @ click.option('--city', default="all", help='要计算的城市')
|
|
# #command='python main.py --city {{dag_run.conf["city"] if "city" in dag_run.conf}} else all',
|
|
# command='echo "this is a test message shown from within the container',
|
|
# mounts=[
|
|
# Mount(source="/var/run/docker.sock", target="/var/run/docker.sock", type="bind")
|
|
# ],
|
|
# task_id='comm_add'
|
|
# )
|
|
# t1 |