优化融合检测

This commit is contained in:
shenjian@fangjia.com
2023-03-14 09:46:19 +08:00
parent 071990bf85
commit d327ad1324
3 changed files with 10 additions and 19 deletions
View File
+7 -16
View File
@@ -1,30 +1,21 @@
#!/usr/bin/env python
# coding: utf-8
# In[56]:
from goojia_utils import dateutils, ftputils, tarutils
import datetime
import calendar
import requests
import json
import shutil
import os
import sys
result = []
# #### 检测传输压缩文件是否完整
# ##### 1. 融合每天检测
try:
today = dateutils.add_months(datetime.datetime.now(), days=1)
date = dateutils.format_date(today, '%Y%m%d')
date = sys.argv[1] if len(sys.argv) > 1 else None
if date is None:
today = dateutils.add_months(datetime.datetime.now(), days=1)
date = dateutils.format_date(today, '%Y%m%d')
print('当前月:' + date)
curr_month_days = calendar.monthrange(today.year, today.month)[1]
print('当月天数:' + str(curr_month_days))
# curr_month_days = calendar.monthrange(today.year, today.month)[1]
# print('当月天数:' + str(curr_month_days))
# In[79]:
+3 -3
View File
@@ -6,12 +6,12 @@ with DAG(
dag_id="merge_dag",
# 每日下午4点
schedule_interval="0 16 * * *",
start_date=pendulum.datetime(2022, 11, 15, tz="Asia/Shanghai"),
start_date=pendulum.datetime(2023, 3, 14, tz="Asia/Shanghai"),
catchup=True,
description="每日融合文件检测"
description="每日融合文件检测, 若想指定检测某一天如20230314,则传参day=20230314"
) as dag:
t1 = BashOperator(
task_id='merge',
bash_command='python /opt/airflow/dags/repo/dags/jh/merge/merge.py',
bash_command='python /opt/airflow/dags/repo/dags/jh/merge/merge.py {{dag_run.conf["day"] if "day" in dag_run.conf}}',
)
t1