diff --git a/dags/business/business/apiuser_notice.py b/dags/business/business/apiuser_notice.py index b9425b5..16126d6 100644 --- a/dags/business/business/apiuser_notice.py +++ b/dags/business/business/apiuser_notice.py @@ -74,7 +74,7 @@ def sendDingDing(text): text = '【Api接口使用提醒】' + text # robot_password = "SECf69d6dc5be4ca7e52b598d7eac75eb2e7d3bd6707c9f0a3ccb3b8a68ecf6f3fc" # robot_webhook = "https://oapi.dingtalk.com/robot/send?access_token=f617e5b73808597d38101393b34c609cbe5353f9ed9a70cabda0b9e6e8ca371e" - token = '1e25f620ea2be926b2441e4f30d67f37abdff2e14088b03b41b4082b11fc34ea'; + token = '1e25f620ea2be926b2441e4f30d67f37abdff2e14088b03b41b4082b11fc34ea' # 发送钉钉群消息 dingutils.send_text(text,token) diff --git a/dags/jh/dag_samples_notice.py b/dags/jh/dag_samples_notice.py new file mode 100644 index 0000000..3dbcaf0 --- /dev/null +++ b/dags/jh/dag_samples_notice.py @@ -0,0 +1,17 @@ +from airflow.operators.bash import BashOperator +import pendulum +from airflow import DAG + +with DAG( + dag_id="samples_notice_dag", + # 每月1日1点运行 + schedule_interval="0 1 1 * *", + start_date=pendulum.datetime(2023, 8, 1, tz="Asia/Shanghai"), + catchup=True, + description="样本数量监控提醒" +) as dag: + t1 = BashOperator( + task_id='samples_notice', + bash_command='python /opt/airflow/dags/repo/dags/jh/sample/samples_notice.py', + ) + t1 \ No newline at end of file diff --git a/dags/jh/sample/__init__.py b/dags/jh/sample/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dags/jh/sample/samples_notice.py b/dags/jh/sample/samples_notice.py new file mode 100644 index 0000000..e466275 --- /dev/null +++ b/dags/jh/sample/samples_notice.py @@ -0,0 +1,130 @@ +from goojia_utils import ElasticSearchClient,dingutils +import pandas as pd +import datetime +import arrow + +def getTime(flag, dayhourminute): + ''' + 获取几小时之前,几分钟前,几天前,几个月前,及几年前的具体时间 + :param flag: 1:天;2:小时;3:分钟;4:月,5:年 + :param dayhourminute: 整数值 + :return: 具体时间 %Y-%m-%d %H:%M:%S + ''' + tn = datetime.datetime.now() + t = None + ttime = '' + if flag <= 3: + if flag == 1: + t = datetime.timedelta(days=dayhourminute) + elif flag == 2: + t = datetime.timedelta(hours=dayhourminute) + elif flag == 3: + t = datetime.timedelta(minutes=dayhourminute) + strtime = tn - t + ttime = strtime.strftime('%Y-%m-%d %H:%M:%S') + else: + dt = arrow.now() + if flag == 4: + ttime = dt.shift(months=-dayhourminute).format("YYYYMMDD") + elif flag == 5: + ttime = str( + int(datetime.datetime.now().strftime("%Y")) - dayhourminute) + "-" + datetime.datetime.now().strftime( + "%m-%d") + return ttime +# 获取当前日期和半年前的日期,统计半年样本数据 +now = datetime.datetime.today() +today = now.strftime('%Y%m%d') +yesterday = getTime(4,6) + +# 区域维度,统计半年内样本数据量 +es206_client = ElasticSearchClient(host='192.168.0.206') +sql = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >='+str(yesterday)+' and p_date<'+str(today) +df = es206_client.scan_by_sql_to_data_frame(sql) + +df_region = pd.DataFrame(df.groupby(by=['city','region'])['_id'].count()).reset_index() +df_region = df_region.rename(columns={'_id':'count'}) +df_region_res = df_region[df_region['count']<20] + +if df_region_res.empty == False: + text = '【房源样本数量自动检测】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情' + +# 计算2个月的日期,用来计算网站数据量减少百分比 +# 当前时间 +now = datetime.datetime.today() +# 获取当前月份的第一天 +first_day_of_month = datetime.datetime(now.year,now.month, 1) +# 获取1个月前的第一天 +last_month = first_day_of_month - datetime.timedelta(days=1) +last_day_of_month = last_month.replace(day=1) +# 获取2个月前的第一天 +last2_month = last_day_of_month - datetime.timedelta(days=1) +last2_day_of_month = last2_month.replace(day=1) + +strat_time = last_day_of_month.strftime('%Y%m%d') +end_time = first_day_of_month.strftime('%Y%m%d') + +strat_time2 = last2_day_of_month.strftime('%Y%m%d') +end_time2 = last_day_of_month.strftime('%Y%m%d') + +# 房源网站维度,按当月数量,和当月数量相比上月减少百分比 +sql1 = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >='+strat_time+' and p_date<='+end_time +sql2 = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >='+strat_time2+' and p_date<='+end_time2 +df_root1 = es206_client.scan_by_sql_to_data_frame(sql1) +df_root2 = es206_client.scan_by_sql_to_data_frame(sql2) + + +df_root_group1 = pd.DataFrame(df_root1.groupby(by=['root'])['_id'].count()).reset_index() +df_root_group2 = pd.DataFrame(df_root2.groupby(by=['root'])['_id'].count()).reset_index() + +df_root_group1 = df_root_group1.rename(columns={'_id':'count'}) +df_root_group2 = df_root_group2.rename(columns={'_id':'count'}) + +# 房源网站,数量维度,数量少于100 +df_root_res1 = df_root_group1[df_root_group1['count']<100] + +if df_root_res1.empty == False: + if text == '': + text = '【房源样本数量自动检测】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情' + +# 房源网站,百分比维度,减少30% +df_root_merge = pd.merge(df_root_group1,df_root_group2,how='left',on=['root']) +df_root_res = df_root_merge.drop(df_root_merge[~((df_root_merge['count_x']0.3))].index) + +if df_root_res.empty == False: + if text == '': + text = '【房源样本数量自动检测】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情' + +path = "/home/airflow/data/jh/price" +file_name = f"{today}房源样本数量检测.xlsx" +air_file_name = path + '/' + file_name + +def up_to_nextcloud(): + """ + 上传统计信息至nextcloud + """ + from nextcloud import NextCloud + NEXTCLOUD_URL = "https://nextcloud.fangjia.com" + NEXTCLOUD_USERNAME = "shenjian@fangjia.com" + NEXTCLOUD_PASSWORD = "goojia123456" + nextcloud_file_name = '/共享/建行/' + file_name + nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD) + nxc.upload_file(air_file_name, nextcloud_file_name) + +if text != '': + with pd.ExcelWriter(air_file_name) as writer: + if df_region_res.empty == False: + df_region_res.to_excel(writer, sheet_name="房源样本数量检测-区域维度", columns=["city","region",'count'], index=False) + if df_root_res1.empty == False: + df_root_res1.to_excel(writer, sheet_name="房源样本数量检测-房源网站维度(样本数)", columns=['root','count_x','count_y'], index=False) + if df_root_res.empty == False: + df_root_res.to_excel(writer, sheet_name="房源样本数量检测-房源网站维度(样本减少百分比)", columns=['root','count_x','count_y'], index=False) + up_to_nextcloud() + token = '1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736' + dingutils.send_text(text,token) + + + + + + +