新增API接口监控提醒任务

This commit is contained in:
zhaojixiang@fangjia.com
2023-07-10 17:06:30 +08:00
parent 8fd13e2dd8
commit 2723cb6ef0
2 changed files with 106 additions and 0 deletions
+89
View File
@@ -0,0 +1,89 @@
from goojia_utils import LOG_MONGO_CLIENT
from sqlalchemy import create_engine
from dingtalkchatbot.chatbot import DingtalkChatbot
import datetime
import pandas as pd
# mysql连接
engine = create_engine('mysql+pymysql://sa:G00jia@192.168.0.42:4306/fangjia?charset=utf8')
# mongodb连接
open_uri_log_conn = LOG_MONGO_CLIENT.get_collection('business', 'open_uri_log')
# 计算查询时间范围
now = datetime.datetime.now()
last = now - datetime.timedelta(hours=1)
now_year = now.year
now_month = now.month
now_day = now.day
now_hour = now.hour
last_year = last.year
last_month = last.month
last_day = last.day
last_hour = last.hour
def sync():
user_ids = []
df_mysql = pd.DataFrame(pd.read_sql("SELECT ID as user_id, IS_TEST FROM api_user WHERE IS_MONITOR = 1", con=engine))
for index in df_mysql.index:
user_id = df_mysql['user_id'].get(index)
user_ids.append(str(user_id))
df_mongo = pd.DataFrame(list(open_uri_log_conn.find(
{'user_id': {'$in': user_ids},
'c_date':
{'$gte': datetime.datetime(last_year, last_month, last_day, last_hour, 0),
'$lt': datetime.datetime(now_year, now_month, now_day, now_hour, 0)
}
}
).batch_size(10000)))
if len(df_mongo) > 0:
df_mongo = df_mongo.groupby(by=['user_id', 'user_name', 'request_uri', 'company_name']).agg(
{'_id': 'count', 'time': 'sum'}).reset_index()
df_mongo['user_id'] = df_mongo['user_id'].astype('int')
df = pd.merge(df_mongo, df_mysql, how='left', on=['user_id'])
df
text = ''
user_id_last = ''
for index in df.index:
user_id = df['user_id'].get(index)
request_uri = df['request_uri'].get(index)
count = df['_id'].get(index)
total_time = df['time'].get(index)
if user_id_last == user_id:
text = text + ' 使用的接口: ' + request_uri + ' 使用次数: ' + str(count) + ' 总计耗时: ' + str(total_time)
else:
user_name = df['user_name'].get(index)
company_name = df['company_name'].get(index)
if df['IS_TEST'].get(index) == 0:
user_type = '正式账户'
else:
user_type = '测试账户'
text = text + '\n' + company_name + ' 用户ID: ' + str(
user_id) + '(' + user_type + ')' + ' 用户名: ' + user_name + ' 使用的接口: ' + request_uri + ' 使用次数: ' + str(
count) + ' 总计耗时: ' + str(total_time)
user_id_last = user_id
sendDingDing(text)
def sendDingDing(text):
text = '【Api接口使用提醒】' + text
# robot_password = "SECf69d6dc5be4ca7e52b598d7eac75eb2e7d3bd6707c9f0a3ccb3b8a68ecf6f3fc"
# robot_webhook = "https://oapi.dingtalk.com/robot/send?access_token=f617e5b73808597d38101393b34c609cbe5353f9ed9a70cabda0b9e6e8ca371e"
robot_password = "";
robot_webhook = "https://oapi.dingtalk.com/robot/send?access_token=1e25f620ea2be926b2441e4f30d67f37abdff2e14088b03b41b4082b11fc34ea"
# 创建机器人
robot = DingtalkChatbot(robot_webhook, robot_password)
# 发送消息
robot.send_text(text)
sync()
+17
View File
@@ -0,0 +1,17 @@
from airflow.operators.bash import BashOperator
import pendulum
from airflow import DAG
with DAG(
dag_id="apiuser_notice_dag",
# 每小时运行
schedule_interval="0 * * * * ",
start_date=pendulum.datetime(2023, 7, 10, tz="Asia/Shanghai"),
catchup=True,
description="API接口监控提醒"
) as dag:
t1 = BashOperator(
task_id='apiuser_notice',
bash_command='python /opt/airflow/dags/repo/dags/business/business/apiuser_notice.py',
)
t1