新增房源样本监控脚本
This commit is contained in:
@@ -74,7 +74,7 @@ def sendDingDing(text):
|
||||
text = '【Api接口使用提醒】' + text
|
||||
# robot_password = "SECf69d6dc5be4ca7e52b598d7eac75eb2e7d3bd6707c9f0a3ccb3b8a68ecf6f3fc"
|
||||
# robot_webhook = "https://oapi.dingtalk.com/robot/send?access_token=f617e5b73808597d38101393b34c609cbe5353f9ed9a70cabda0b9e6e8ca371e"
|
||||
token = '1e25f620ea2be926b2441e4f30d67f37abdff2e14088b03b41b4082b11fc34ea';
|
||||
token = '1e25f620ea2be926b2441e4f30d67f37abdff2e14088b03b41b4082b11fc34ea'
|
||||
# 发送钉钉群消息
|
||||
dingutils.send_text(text,token)
|
||||
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
from airflow.operators.bash import BashOperator
|
||||
import pendulum
|
||||
from airflow import DAG
|
||||
|
||||
with DAG(
|
||||
dag_id="samples_notice_dag",
|
||||
# 每月1日1点运行
|
||||
schedule_interval="0 1 1 * *",
|
||||
start_date=pendulum.datetime(2023, 8, 1, tz="Asia/Shanghai"),
|
||||
catchup=True,
|
||||
description="样本数量监控提醒"
|
||||
) as dag:
|
||||
t1 = BashOperator(
|
||||
task_id='samples_notice',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/jh/sample/samples_notice.py',
|
||||
)
|
||||
t1
|
||||
@@ -0,0 +1,130 @@
|
||||
from goojia_utils import ElasticSearchClient,dingutils
|
||||
import pandas as pd
|
||||
import datetime
|
||||
import arrow
|
||||
|
||||
def getTime(flag, dayhourminute):
|
||||
'''
|
||||
获取几小时之前,几分钟前,几天前,几个月前,及几年前的具体时间
|
||||
:param flag: 1:天;2:小时;3:分钟;4:月,5:年
|
||||
:param dayhourminute: 整数值
|
||||
:return: 具体时间 %Y-%m-%d %H:%M:%S
|
||||
'''
|
||||
tn = datetime.datetime.now()
|
||||
t = None
|
||||
ttime = ''
|
||||
if flag <= 3:
|
||||
if flag == 1:
|
||||
t = datetime.timedelta(days=dayhourminute)
|
||||
elif flag == 2:
|
||||
t = datetime.timedelta(hours=dayhourminute)
|
||||
elif flag == 3:
|
||||
t = datetime.timedelta(minutes=dayhourminute)
|
||||
strtime = tn - t
|
||||
ttime = strtime.strftime('%Y-%m-%d %H:%M:%S')
|
||||
else:
|
||||
dt = arrow.now()
|
||||
if flag == 4:
|
||||
ttime = dt.shift(months=-dayhourminute).format("YYYYMMDD")
|
||||
elif flag == 5:
|
||||
ttime = str(
|
||||
int(datetime.datetime.now().strftime("%Y")) - dayhourminute) + "-" + datetime.datetime.now().strftime(
|
||||
"%m-%d")
|
||||
return ttime
|
||||
# 获取当前日期和半年前的日期,统计半年样本数据
|
||||
now = datetime.datetime.today()
|
||||
today = now.strftime('%Y%m%d')
|
||||
yesterday = getTime(4,6)
|
||||
|
||||
# 区域维度,统计半年内样本数据量
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
sql = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >='+str(yesterday)+' and p_date<'+str(today)
|
||||
df = es206_client.scan_by_sql_to_data_frame(sql)
|
||||
|
||||
df_region = pd.DataFrame(df.groupby(by=['city','region'])['_id'].count()).reset_index()
|
||||
df_region = df_region.rename(columns={'_id':'count'})
|
||||
df_region_res = df_region[df_region['count']<20]
|
||||
|
||||
if df_region_res.empty == False:
|
||||
text = '【房源样本数量自动检测】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情'
|
||||
|
||||
# 计算2个月的日期,用来计算网站数据量减少百分比
|
||||
# 当前时间
|
||||
now = datetime.datetime.today()
|
||||
# 获取当前月份的第一天
|
||||
first_day_of_month = datetime.datetime(now.year,now.month, 1)
|
||||
# 获取1个月前的第一天
|
||||
last_month = first_day_of_month - datetime.timedelta(days=1)
|
||||
last_day_of_month = last_month.replace(day=1)
|
||||
# 获取2个月前的第一天
|
||||
last2_month = last_day_of_month - datetime.timedelta(days=1)
|
||||
last2_day_of_month = last2_month.replace(day=1)
|
||||
|
||||
strat_time = last_day_of_month.strftime('%Y%m%d')
|
||||
end_time = first_day_of_month.strftime('%Y%m%d')
|
||||
|
||||
strat_time2 = last2_day_of_month.strftime('%Y%m%d')
|
||||
end_time2 = last_day_of_month.strftime('%Y%m%d')
|
||||
|
||||
# 房源网站维度,按当月数量,和当月数量相比上月减少百分比
|
||||
sql1 = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >='+strat_time+' and p_date<='+end_time
|
||||
sql2 = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >='+strat_time2+' and p_date<='+end_time2
|
||||
df_root1 = es206_client.scan_by_sql_to_data_frame(sql1)
|
||||
df_root2 = es206_client.scan_by_sql_to_data_frame(sql2)
|
||||
|
||||
|
||||
df_root_group1 = pd.DataFrame(df_root1.groupby(by=['root'])['_id'].count()).reset_index()
|
||||
df_root_group2 = pd.DataFrame(df_root2.groupby(by=['root'])['_id'].count()).reset_index()
|
||||
|
||||
df_root_group1 = df_root_group1.rename(columns={'_id':'count'})
|
||||
df_root_group2 = df_root_group2.rename(columns={'_id':'count'})
|
||||
|
||||
# 房源网站,数量维度,数量少于100
|
||||
df_root_res1 = df_root_group1[df_root_group1['count']<100]
|
||||
|
||||
if df_root_res1.empty == False:
|
||||
if text == '':
|
||||
text = '【房源样本数量自动检测】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情'
|
||||
|
||||
# 房源网站,百分比维度,减少30%
|
||||
df_root_merge = pd.merge(df_root_group1,df_root_group2,how='left',on=['root'])
|
||||
df_root_res = df_root_merge.drop(df_root_merge[~((df_root_merge['count_x']<df_root_merge['count_y']) & ((df_root_merge['count_y']-df_root_merge['count_x'])/df_root_merge['count_y']>0.3))].index)
|
||||
|
||||
if df_root_res.empty == False:
|
||||
if text == '':
|
||||
text = '【房源样本数量自动检测】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情'
|
||||
|
||||
path = "/home/airflow/data/jh/price"
|
||||
file_name = f"{today}房源样本数量检测.xlsx"
|
||||
air_file_name = path + '/' + file_name
|
||||
|
||||
def up_to_nextcloud():
|
||||
"""
|
||||
上传统计信息至nextcloud
|
||||
"""
|
||||
from nextcloud import NextCloud
|
||||
NEXTCLOUD_URL = "https://nextcloud.fangjia.com"
|
||||
NEXTCLOUD_USERNAME = "shenjian@fangjia.com"
|
||||
NEXTCLOUD_PASSWORD = "goojia123456"
|
||||
nextcloud_file_name = '/共享/建行/' + file_name
|
||||
nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD)
|
||||
nxc.upload_file(air_file_name, nextcloud_file_name)
|
||||
|
||||
if text != '':
|
||||
with pd.ExcelWriter(air_file_name) as writer:
|
||||
if df_region_res.empty == False:
|
||||
df_region_res.to_excel(writer, sheet_name="房源样本数量检测-区域维度", columns=["city","region",'count'], index=False)
|
||||
if df_root_res1.empty == False:
|
||||
df_root_res1.to_excel(writer, sheet_name="房源样本数量检测-房源网站维度(样本数)", columns=['root','count_x','count_y'], index=False)
|
||||
if df_root_res.empty == False:
|
||||
df_root_res.to_excel(writer, sheet_name="房源样本数量检测-房源网站维度(样本减少百分比)", columns=['root','count_x','count_y'], index=False)
|
||||
up_to_nextcloud()
|
||||
token = '1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736'
|
||||
dingutils.send_text(text,token)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user