From 034a2a80ba140fac05832285fc264ef755129e32 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Fri, 17 Mar 2023 21:01:37 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=97=A5=E6=9C=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 3 ++- .../{amap_key_generate_dag.py => dag_amap_key_generate.py} | 0 ..._to_old_seaweed_dag.py => dag_change_new_to_old_seaweed.py} | 0 .../{community_summary_dag.py => dag_community_summary.py} | 0 dags/business/{didata_sync_dag.py => dag_didata_sync.py} | 2 +- dags/jh/{merge_dag.py => dag_merge.py} | 2 +- dags/jh/{price_exception_dag.py => dag_price_exception.py} | 2 +- dags/jh/{third_price_dag.py => dag_third_price.py} | 0 dags/mt/{mt_dag.py => dag_mt.py} | 0 .../{district_facilities_dag.py => dag_district_facilities.py} | 2 +- ...unt_price_rent_dag.py => dag_community_count_price_rent.py} | 2 +- .../{district_sell_deal_dag.py => dag_district_sell_deal.py} | 2 +- 12 files changed, 8 insertions(+), 7 deletions(-) rename dags/business/{amap_key_generate_dag.py => dag_amap_key_generate.py} (100%) rename dags/business/{change_new_to_old_seaweed_dag.py => dag_change_new_to_old_seaweed.py} (100%) rename dags/business/{community_summary_dag.py => dag_community_summary.py} (100%) rename dags/business/{didata_sync_dag.py => dag_didata_sync.py} (89%) rename dags/jh/{merge_dag.py => dag_merge.py} (88%) rename dags/jh/{price_exception_dag.py => dag_price_exception.py} (95%) rename dags/jh/{third_price_dag.py => dag_third_price.py} (100%) rename dags/mt/{mt_dag.py => dag_mt.py} (100%) rename dags/poi/{district_facilities_dag.py => dag_district_facilities.py} (88%) rename dags/stat/{count_price_rent_dag.py => dag_community_count_price_rent.py} (85%) rename dags/stat/{district_sell_deal_dag.py => dag_district_sell_deal.py} (91%) diff --git a/README.md b/README.md index 5ba9e66..9190101 100644 --- a/README.md +++ b/README.md @@ -7,4 +7,5 @@ pip install -r requirements.txt -i https://repo.fangjia.com/repository/pypi-publ - 具体业务逻辑放入xxx/business中,如poi/business - 添加.airflowignore xxx/business/*, 如poi/business/* -- dag放在xxx目录下即可,命名为xx_dag.py \ No newline at end of file +- dag放在xxx目录下即可,命名为dag_xx.py,之前命名为xx_dag导致xx与报名相同而加入airflowignore后dag不显示问题, + 如merge_dag会不显示,而dag_merge即可,怀疑是jh/merge/*的bug \ No newline at end of file diff --git a/dags/business/amap_key_generate_dag.py b/dags/business/dag_amap_key_generate.py similarity index 100% rename from dags/business/amap_key_generate_dag.py rename to dags/business/dag_amap_key_generate.py diff --git a/dags/business/change_new_to_old_seaweed_dag.py b/dags/business/dag_change_new_to_old_seaweed.py similarity index 100% rename from dags/business/change_new_to_old_seaweed_dag.py rename to dags/business/dag_change_new_to_old_seaweed.py diff --git a/dags/business/community_summary_dag.py b/dags/business/dag_community_summary.py similarity index 100% rename from dags/business/community_summary_dag.py rename to dags/business/dag_community_summary.py diff --git a/dags/business/didata_sync_dag.py b/dags/business/dag_didata_sync.py similarity index 89% rename from dags/business/didata_sync_dag.py rename to dags/business/dag_didata_sync.py index 01fb1a8..4ac3218 100644 --- a/dags/business/didata_sync_dag.py +++ b/dags/business/dag_didata_sync.py @@ -6,7 +6,7 @@ with DAG( "didata_sync_dag", # 每月28日3点整分运行 schedule_interval="0 3 28 * *", - start_date=pendulum.datetime(2022, 8, 22, tz="Asia/Shanghai"), + start_date=pendulum.datetime(2023, 3, 17, tz="Asia/Shanghai"), catchup=True, description="地数相关数据同步type: district或office, 不填默认同步全部" ) as dag: diff --git a/dags/jh/merge_dag.py b/dags/jh/dag_merge.py similarity index 88% rename from dags/jh/merge_dag.py rename to dags/jh/dag_merge.py index 22bb27b..d1b3d48 100644 --- a/dags/jh/merge_dag.py +++ b/dags/jh/dag_merge.py @@ -6,7 +6,7 @@ with DAG( dag_id="merge_dag", # 每日下午4点 schedule_interval="0 16 * * *", - start_date=pendulum.datetime(2023, 3, 15, tz="Asia/Shanghai"), + start_date=pendulum.datetime(2023, 3, 17, tz="Asia/Shanghai"), catchup=True, description="每日融合文件检测, 若想指定检测某一天如20230314,则传参day='20230314'" ) as dag: diff --git a/dags/jh/price_exception_dag.py b/dags/jh/dag_price_exception.py similarity index 95% rename from dags/jh/price_exception_dag.py rename to dags/jh/dag_price_exception.py index 68664ee..2ba443c 100644 --- a/dags/jh/price_exception_dag.py +++ b/dags/jh/dag_price_exception.py @@ -6,7 +6,7 @@ with DAG( dag_id="price_exception_dag", # 每月23日运行 schedule_interval="0 10 23 * *", - start_date=pendulum.datetime(2022, 8, 23, tz="Asia/Shanghai"), + start_date=pendulum.datetime(2023, 3, 17, tz="Asia/Shanghai"), catchup=True, description="建行每月价格异常检测脚本,默认检测本月与上月差异, 若对比上月与前月价格,设置year_month=\"202210\"(前月日期即可) 若输出样本存在与否,设置year_month=sample" ) as dag: diff --git a/dags/jh/third_price_dag.py b/dags/jh/dag_third_price.py similarity index 100% rename from dags/jh/third_price_dag.py rename to dags/jh/dag_third_price.py diff --git a/dags/mt/mt_dag.py b/dags/mt/dag_mt.py similarity index 100% rename from dags/mt/mt_dag.py rename to dags/mt/dag_mt.py diff --git a/dags/poi/district_facilities_dag.py b/dags/poi/dag_district_facilities.py similarity index 88% rename from dags/poi/district_facilities_dag.py rename to dags/poi/dag_district_facilities.py index 54ce1d5..a198ac8 100644 --- a/dags/poi/district_facilities_dag.py +++ b/dags/poi/dag_district_facilities.py @@ -11,7 +11,7 @@ with DAG( "district_facilities_dag", # 每月1日1点运行 schedule_interval="0 1 1 * *", - start_date=pendulum.datetime(2022, 6, 18, tz="Asia/Shanghai"), + start_date=pendulum.datetime(2023, 3, 17, tz="Asia/Shanghai"), catchup=True, ) as dag: t1 = BashOperator( diff --git a/dags/stat/count_price_rent_dag.py b/dags/stat/dag_community_count_price_rent.py similarity index 85% rename from dags/stat/count_price_rent_dag.py rename to dags/stat/dag_community_count_price_rent.py index fafd553..ce27bba 100644 --- a/dags/stat/count_price_rent_dag.py +++ b/dags/stat/dag_community_count_price_rent.py @@ -6,7 +6,7 @@ with DAG( "count_price_rent_dag", # 每周日运行 schedule_interval="0 10 * * 7", - start_date=pendulum.datetime(2022, 7, 1, tz="Asia/Shanghai"), + start_date=pendulum.datetime(2023, 3, 17, tz="Asia/Shanghai"), catchup=True, ) as dag: t1 = BashOperator( diff --git a/dags/stat/district_sell_deal_dag.py b/dags/stat/dag_district_sell_deal.py similarity index 91% rename from dags/stat/district_sell_deal_dag.py rename to dags/stat/dag_district_sell_deal.py index 423242c..d21ddd9 100644 --- a/dags/stat/district_sell_deal_dag.py +++ b/dags/stat/dag_district_sell_deal.py @@ -6,7 +6,7 @@ with DAG( "district_sell_deal_dag", # 每周五运行 schedule_interval="0 1 * * 5", - start_date=pendulum.datetime(2022, 6, 20, tz="Asia/Shanghai"), + start_date=pendulum.datetime(2023, 3, 17, tz="Asia/Shanghai"), catchup=True, ) as dag: t1 = BashOperator(