diff --git a/dags/jh/dag_samples_notice.py b/dags/jh/dag_samples_notice.py index 3dbcaf0..0f2a546 100644 --- a/dags/jh/dag_samples_notice.py +++ b/dags/jh/dag_samples_notice.py @@ -4,8 +4,8 @@ from airflow import DAG with DAG( dag_id="samples_notice_dag", - # 每月1日1点运行 - schedule_interval="0 1 1 * *", + # 每月10日1点运行 + schedule_interval="0 0 10 * *", start_date=pendulum.datetime(2023, 8, 1, tz="Asia/Shanghai"), catchup=True, description="样本数量监控提醒" diff --git a/dags/jh/sample/samples_notice.py b/dags/jh/sample/samples_notice.py index e466275..c561298 100644 --- a/dags/jh/sample/samples_notice.py +++ b/dags/jh/sample/samples_notice.py @@ -1,44 +1,16 @@ -from goojia_utils import ElasticSearchClient,dingutils +from goojia_utils import ElasticSearchClient,dingutils,dateutils import pandas as pd import datetime -import arrow -def getTime(flag, dayhourminute): - ''' - 获取几小时之前,几分钟前,几天前,几个月前,及几年前的具体时间 - :param flag: 1:天;2:小时;3:分钟;4:月,5:年 - :param dayhourminute: 整数值 - :return: 具体时间 %Y-%m-%d %H:%M:%S - ''' - tn = datetime.datetime.now() - t = None - ttime = '' - if flag <= 3: - if flag == 1: - t = datetime.timedelta(days=dayhourminute) - elif flag == 2: - t = datetime.timedelta(hours=dayhourminute) - elif flag == 3: - t = datetime.timedelta(minutes=dayhourminute) - strtime = tn - t - ttime = strtime.strftime('%Y-%m-%d %H:%M:%S') - else: - dt = arrow.now() - if flag == 4: - ttime = dt.shift(months=-dayhourminute).format("YYYYMMDD") - elif flag == 5: - ttime = str( - int(datetime.datetime.now().strftime("%Y")) - dayhourminute) + "-" + datetime.datetime.now().strftime( - "%m-%d") - return ttime # 获取当前日期和半年前的日期,统计半年样本数据 now = datetime.datetime.today() -today = now.strftime('%Y%m%d') -yesterday = getTime(4,6) +now_day = now.strftime('%Y%m%d') +last = dateutils.add_months(now,0,-6,0,0) +last_day = last.strftime('%Y%m%d') # 区域维度,统计半年内样本数据量 es206_client = ElasticSearchClient(host='192.168.0.206') -sql = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >='+str(yesterday)+' and p_date<'+str(today) +sql = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >='+str(last_day)+' and p_date<'+str(now_day) df = es206_client.scan_by_sql_to_data_frame(sql) df_region = pd.DataFrame(df.groupby(by=['city','region'])['_id'].count()).reset_index() @@ -95,7 +67,7 @@ if df_root_res.empty == False: text = '【房源样本数量自动检测】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情' path = "/home/airflow/data/jh/price" -file_name = f"{today}房源样本数量检测.xlsx" +file_name = f"{now_day}房源样本数量检测.xlsx" air_file_name = path + '/' + file_name def up_to_nextcloud():