This commit is contained in:
沈健
2024-08-14 18:09:40 +08:00
parent 7e60e06c3f
commit 1f60c344bd
2 changed files with 147 additions and 147 deletions
+17 -17
View File
@@ -1,17 +1,17 @@
from airflow.operators.bash import BashOperator
import pendulum
from airflow import DAG
with DAG(
dag_id="samples_notice_dag",
# 每月10日1点运行
schedule_interval="0 0 10 * *",
start_date=pendulum.datetime(2023, 8, 1, tz="Asia/Shanghai"),
catchup=True,
description="样本数量监控提醒"
) as dag:
t1 = BashOperator(
task_id='samples_notice',
bash_command='python /opt/airflow/dags/repo/dags/jh/sample/samples_notice.py',
)
t1
# from airflow.operators.bash import BashOperator
# import pendulum
# from airflow import DAG
#
# with DAG(
# dag_id="samples_notice_dag",
# # 每月10日1点运行
# schedule_interval="0 0 10 * *",
# start_date=pendulum.datetime(2023, 8, 1, tz="Asia/Shanghai"),
# catchup=True,
# description="样本数量监控提醒"
# ) as dag:
# t1 = BashOperator(
# task_id='samples_notice',
# bash_command='python /opt/airflow/dags/repo/dags/jh/sample/samples_notice.py',
# )
# t1
+130 -130
View File
@@ -1,130 +1,130 @@
from goojia_utils import ElasticSearchClient, dingutils, dateutils
import pandas as pd
import datetime
import logging
log = logging.getLogger(__name__)
# 获取当前日期和2个月前的日期,统计2个月样本数据
now = datetime.datetime.today()
first_day = datetime.datetime(now.year, now.month, 1)
now_day = first_day.strftime('%Y%m%d')
last = dateutils.add_months(first_day, 0, -2, 0, 0)
last_day = last.strftime('%Y%m%d')
log.info(f"系统当前时间 : {now}")
def proc():
# 区域维度,统计2个月内样本数据量
es206_client = ElasticSearchClient(host='192.168.0.206')
sql = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >=' + str(last_day) + ' and p_date<' + str(now_day)
log.info(f"城市区域维度 sql : {sql}")
df = es206_client.scan_by_sql_to_data_frame(sql)
df_region = pd.DataFrame(df.groupby(by=['city', 'region'])['_id'].count()).reset_index()
df_region = df_region.rename(columns={'_id': 'count'})
df_region_res = df_region[df_region['count'] < 20]
if df_region_res.empty == False:
text = '【房源样本数量自动检测】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情'
# 计算2个月的日期,用来计算网站数据量减少百分比
# 当前时间
now = datetime.datetime.today()
# 获取当前月份的第一天
first_day_of_month = datetime.datetime(now.year, now.month, 1)
# 获取1个月前的第一天
last_month = first_day_of_month - datetime.timedelta(days=1)
last_day_of_month = last_month.replace(day=1)
# 获取2个月前的第一天
last2_month = last_day_of_month - datetime.timedelta(days=1)
last2_day_of_month = last2_month.replace(day=1)
strat_time = last_day_of_month.strftime('%Y%m%d')
end_time = first_day_of_month.strftime('%Y%m%d')
strat_time2 = last2_day_of_month.strftime('%Y%m%d')
end_time2 = last_day_of_month.strftime('%Y%m%d')
# 房源网站维度,按当月数量,和当月数量相比上月减少百分比
sql1 = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >=' + strat_time + ' and p_date<=' + end_time
sql2 = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >=' + strat_time2 + ' and p_date<=' + end_time2
log.info(f"房源网站维度-百分比 当月sql : {sql1}")
log.info(f"房源网站维度-百分比 上月sql : {sql2}")
df_root1 = es206_client.scan_by_sql_to_data_frame(sql1)
df_root2 = es206_client.scan_by_sql_to_data_frame(sql2)
df_root_group1 = pd.DataFrame(df_root1.groupby(by=['root'])['_id'].count()).reset_index()
df_root_group2 = pd.DataFrame(df_root2.groupby(by=['root'])['_id'].count()).reset_index()
df_root_group1 = df_root_group1.rename(columns={'_id': 'count'})
df_root_group2 = df_root_group2.rename(columns={'_id': 'count'})
# 房源网站,百分比维度,减少30%
df_root_merge = pd.merge(df_root_group1, df_root_group2, how='left', on=['root'])
count_new = 'count_' + strat_time[0:6]
count_old = 'count_' + strat_time2[0:6]
df_root_merge = df_root_merge.rename(columns={'count_x': count_new, 'count_y': count_old})
df_root_res = df_root_merge.drop(df_root_merge[~((df_root_merge[count_new] < df_root_merge[count_old]) & (
(df_root_merge[count_old] - df_root_merge[count_new]) / df_root_merge[count_old] > 0.3))].index)
if df_root_res.empty == False:
if text == '':
text = '【房源样本数量自动检测】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情'
# 房源网站,数量维度,前70个城市当月数量少于100
cities = pd.read_excel("/opt/airflow/dags/repo/dags/jh/para/估值模型测试城市模板.xlsx")
topCities = cities.head(70)
city_list = list(topCities['cityname'])
log.info(f"房源网站维度-数量 city_list : {city_list}")
df_root_city_group1 = pd.DataFrame(df_root1.groupby(by=['root', 'city'])['_id'].count()).reset_index()
df_root_city_group1 = df_root_city_group1.rename(columns={'_id': 'count'})
df_root_city_res = df_root_city_group1[
(df_root_city_group1['city'].isin(city_list)) & (df_root_city_group1['count'] < 100)].reset_index()
if df_root_city_res.empty == False:
if text == '':
text = '【房源样本数量自动检测】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情'
path = "/home/airflow/data/jh/price"
file_name = f"{now_day}房源样本数量检测.xlsx"
air_file_name = path + '/' + file_name
return text, air_file_name, df_region_res, df_root_city_res, df_root_res, file_name, count_old, count_new
text, air_file_name, df_region_res, df_root_city_res, df_root_res, file_name, count_old, count_new = proc()
def up_to_nextcloud():
"""
上传统计信息至nextcloud
"""
from nextcloud import NextCloud
NEXTCLOUD_URL = "https://nextcloud.fangjia.com"
NEXTCLOUD_USERNAME = "shenjian@fangjia.com"
NEXTCLOUD_PASSWORD = "goojia123456"
nextcloud_file_name = '/共享/建行/' + file_name
nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD)
nxc.upload_file(air_file_name, nextcloud_file_name)
if text != '':
with pd.ExcelWriter(air_file_name) as writer:
if not df_region_res.empty:
df_region_res.to_excel(writer, sheet_name="房源样本数量检测-区域维度", columns=["city", "region", 'count'], index=False)
if not df_root_city_res.empty:
df_root_city_res.to_excel(writer, sheet_name="房源样本数量检测-房源网站维度(样本数)", columns=['root', 'city', 'count'],
index=False)
if not df_root_res.empty:
df_root_res.to_excel(writer, sheet_name="房源样本数量检测-房源网站维度(样本减少百分比)", columns=['root', count_old, count_new],
index=False)
up_to_nextcloud()
token = '1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736'
dingutils.send_text(text, token)
# from goojia_utils import ElasticSearchClient, dingutils, dateutils
# import pandas as pd
# import datetime
# import logging
#
# log = logging.getLogger(__name__)
#
# # 获取当前日期和2个月前的日期,统计2个月样本数据
# now = datetime.datetime.today()
# first_day = datetime.datetime(now.year, now.month, 1)
# now_day = first_day.strftime('%Y%m%d')
# last = dateutils.add_months(first_day, 0, -2, 0, 0)
# last_day = last.strftime('%Y%m%d')
#
# log.info(f"系统当前时间 : {now}")
#
#
# def proc():
# # 区域维度,统计2个月内样本数据量
# es206_client = ElasticSearchClient(host='192.168.0.206')
# sql = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >=' + str(last_day) + ' and p_date<' + str(now_day)
#
# log.info(f"城市区域维度 sql : {sql}")
# df = es206_client.scan_by_sql_to_data_frame(sql)
#
# df_region = pd.DataFrame(df.groupby(by=['city', 'region'])['_id'].count()).reset_index()
# df_region = df_region.rename(columns={'_id': 'count'})
# df_region_res = df_region[df_region['count'] < 20]
#
# if df_region_res.empty == False:
# text = '【房源样本数量自动检测】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情'
#
# # 计算2个月的日期,用来计算网站数据量减少百分比
# # 当前时间
# now = datetime.datetime.today()
# # 获取当前月份的第一天
# first_day_of_month = datetime.datetime(now.year, now.month, 1)
# # 获取1个月前的第一天
# last_month = first_day_of_month - datetime.timedelta(days=1)
# last_day_of_month = last_month.replace(day=1)
# # 获取2个月前的第一天
# last2_month = last_day_of_month - datetime.timedelta(days=1)
# last2_day_of_month = last2_month.replace(day=1)
#
# strat_time = last_day_of_month.strftime('%Y%m%d')
# end_time = first_day_of_month.strftime('%Y%m%d')
#
# strat_time2 = last2_day_of_month.strftime('%Y%m%d')
# end_time2 = last_day_of_month.strftime('%Y%m%d')
#
# # 房源网站维度,按当月数量,和当月数量相比上月减少百分比
# sql1 = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >=' + strat_time + ' and p_date<=' + end_time
# sql2 = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >=' + strat_time2 + ' and p_date<=' + end_time2
# log.info(f"房源网站维度-百分比 当月sql : {sql1}")
# log.info(f"房源网站维度-百分比 上月sql : {sql2}")
# df_root1 = es206_client.scan_by_sql_to_data_frame(sql1)
# df_root2 = es206_client.scan_by_sql_to_data_frame(sql2)
#
# df_root_group1 = pd.DataFrame(df_root1.groupby(by=['root'])['_id'].count()).reset_index()
# df_root_group2 = pd.DataFrame(df_root2.groupby(by=['root'])['_id'].count()).reset_index()
#
# df_root_group1 = df_root_group1.rename(columns={'_id': 'count'})
# df_root_group2 = df_root_group2.rename(columns={'_id': 'count'})
#
# # 房源网站,百分比维度,减少30%
# df_root_merge = pd.merge(df_root_group1, df_root_group2, how='left', on=['root'])
#
# count_new = 'count_' + strat_time[0:6]
# count_old = 'count_' + strat_time2[0:6]
#
# df_root_merge = df_root_merge.rename(columns={'count_x': count_new, 'count_y': count_old})
#
# df_root_res = df_root_merge.drop(df_root_merge[~((df_root_merge[count_new] < df_root_merge[count_old]) & (
# (df_root_merge[count_old] - df_root_merge[count_new]) / df_root_merge[count_old] > 0.3))].index)
#
# if df_root_res.empty == False:
# if text == '':
# text = '【房源样本数量自动检测】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情'
#
# # 房源网站,数量维度,前70个城市当月数量少于100
#
# cities = pd.read_excel("/opt/airflow/dags/repo/dags/jh/para/估值模型测试城市模板.xlsx")
# topCities = cities.head(70)
# city_list = list(topCities['cityname'])
# log.info(f"房源网站维度-数量 city_list : {city_list}")
#
# df_root_city_group1 = pd.DataFrame(df_root1.groupby(by=['root', 'city'])['_id'].count()).reset_index()
# df_root_city_group1 = df_root_city_group1.rename(columns={'_id': 'count'})
# df_root_city_res = df_root_city_group1[
# (df_root_city_group1['city'].isin(city_list)) & (df_root_city_group1['count'] < 100)].reset_index()
#
# if df_root_city_res.empty == False:
# if text == '':
# text = '【房源样本数量自动检测】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情'
#
# path = "/home/airflow/data/jh/price"
# file_name = f"{now_day}房源样本数量检测.xlsx"
# air_file_name = path + '/' + file_name
# return text, air_file_name, df_region_res, df_root_city_res, df_root_res, file_name, count_old, count_new
#
#
# text, air_file_name, df_region_res, df_root_city_res, df_root_res, file_name, count_old, count_new = proc()
#
#
# def up_to_nextcloud():
# """
# 上传统计信息至nextcloud
# """
# from nextcloud import NextCloud
# NEXTCLOUD_URL = "https://nextcloud.fangjia.com"
# NEXTCLOUD_USERNAME = "shenjian@fangjia.com"
# NEXTCLOUD_PASSWORD = "goojia123456"
# nextcloud_file_name = '/共享/建行/' + file_name
# nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD)
# nxc.upload_file(air_file_name, nextcloud_file_name)
#
#
# if text != '':
# with pd.ExcelWriter(air_file_name) as writer:
# if not df_region_res.empty:
# df_region_res.to_excel(writer, sheet_name="房源样本数量检测-区域维度", columns=["city", "region", 'count'], index=False)
# if not df_root_city_res.empty:
# df_root_city_res.to_excel(writer, sheet_name="房源样本数量检测-房源网站维度(样本数)", columns=['root', 'city', 'count'],
# index=False)
# if not df_root_res.empty:
# df_root_res.to_excel(writer, sheet_name="房源样本数量检测-房源网站维度(样本减少百分比)", columns=['root', count_old, count_new],
# index=False)
# up_to_nextcloud()
# token = '1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736'
# dingutils.send_text(text, token)