From cda9cd919aca8e29025a3d7cabd19b144423fe9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 14 Aug 2024 16:26:32 +0800 Subject: [PATCH 1/4] =?UTF-8?q?=E6=96=B0=E5=A2=9EOPEN=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E7=9B=91=E6=8E=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/business/business/fangjia_log.py | 24 ++++++++++++++++++++++++ dags/business/dag_fangjia_log.py | 19 +++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 dags/business/business/fangjia_log.py create mode 100644 dags/business/dag_fangjia_log.py diff --git a/dags/business/business/fangjia_log.py b/dags/business/business/fangjia_log.py new file mode 100644 index 0000000..f2d054e --- /dev/null +++ b/dags/business/business/fangjia_log.py @@ -0,0 +1,24 @@ +import logging +from goojia_utils import ElasticSearchClient, dingutils + +log = logging.getLogger(__name__) + +es206_client = ElasticSearchClient(host='192.168.0.206') + +from datetime import datetime, timedelta + +# 获取当前时间 +now = datetime.now() +# 计算当前时间以前2分钟的时间 +two_minutes_ago = now - timedelta(minutes=2) +# 将时间转换为时间戳(以毫秒为单位) +# 注意:timestamp() 方法返回的是秒,所以我们需要乘以1000来得到毫秒 +timestamp_milliseconds = int(two_minutes_ago.timestamp() * 1000) + +count = 0 +for doc in es206_client.scan_by_sql("select code from fangjia_log where code >= 500 and creationDate > " + str(timestamp_milliseconds)): + count += 1 + if count > 2: + print('【OPEN日志监控】最近两分钟超过两条错误,请检查日志!') + dingutils.send_text('最近两分钟超过两条错误,请检查OPEN日志!', 'ab4231000995a8c90d8d1e91dfd6930c3896dd1c3bbf1f0af19e19004cf6c98c') + break \ No newline at end of file diff --git a/dags/business/dag_fangjia_log.py b/dags/business/dag_fangjia_log.py new file mode 100644 index 0000000..5e5d670 --- /dev/null +++ b/dags/business/dag_fangjia_log.py @@ -0,0 +1,19 @@ +from datetime import timedelta + +from airflow.operators.bash import BashOperator +import pendulum +from airflow import DAG + +with DAG( + dag_id="fangjia_log_dag", + # 每2分钟监控 + schedule_interval=timedelta(minutes=2), + start_date=pendulum.datetime(2024, 8, 14, tz="Asia/Shanghai"), + catchup=True, + description="OPEN日志监控" +) as dag: + t1 = BashOperator( + task_id='fangjia_log', + bash_command='python /opt/airflow/dags/repo/dags/business/business/fangjia_log.py', + ) + t1 \ No newline at end of file From 7e60e06c3faa8217454ae6e05ed728c7f1c5a5cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 14 Aug 2024 18:08:10 +0800 Subject: [PATCH 2/4] =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/jh/sample/samples_notice.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dags/jh/sample/samples_notice.py b/dags/jh/sample/samples_notice.py index 6074c3e..ceeb28f 100644 --- a/dags/jh/sample/samples_notice.py +++ b/dags/jh/sample/samples_notice.py @@ -18,8 +18,8 @@ log.info(f"系统当前时间 : {now}") def proc(): # 区域维度,统计2个月内样本数据量 es206_client = ElasticSearchClient(host='192.168.0.206') - sql = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >=' + str( - last_day) + ' and p_date<' + str(now_day) + sql = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >=' + str(last_day) + ' and p_date<' + str(now_day) + log.info(f"城市区域维度 sql : {sql}") df = es206_client.scan_by_sql_to_data_frame(sql) From 1f60c344bdccd92dc21c3db8b32ea18e28535607 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 14 Aug 2024 18:09:40 +0800 Subject: [PATCH 3/4] =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/jh/dag_samples_notice.py | 34 ++-- dags/jh/sample/samples_notice.py | 260 +++++++++++++++---------------- 2 files changed, 147 insertions(+), 147 deletions(-) diff --git a/dags/jh/dag_samples_notice.py b/dags/jh/dag_samples_notice.py index 0f2a546..c00d38a 100644 --- a/dags/jh/dag_samples_notice.py +++ b/dags/jh/dag_samples_notice.py @@ -1,17 +1,17 @@ -from airflow.operators.bash import BashOperator -import pendulum -from airflow import DAG - -with DAG( - dag_id="samples_notice_dag", - # 每月10日1点运行 - schedule_interval="0 0 10 * *", - start_date=pendulum.datetime(2023, 8, 1, tz="Asia/Shanghai"), - catchup=True, - description="样本数量监控提醒" -) as dag: - t1 = BashOperator( - task_id='samples_notice', - bash_command='python /opt/airflow/dags/repo/dags/jh/sample/samples_notice.py', - ) - t1 \ No newline at end of file +# from airflow.operators.bash import BashOperator +# import pendulum +# from airflow import DAG +# +# with DAG( +# dag_id="samples_notice_dag", +# # 每月10日1点运行 +# schedule_interval="0 0 10 * *", +# start_date=pendulum.datetime(2023, 8, 1, tz="Asia/Shanghai"), +# catchup=True, +# description="样本数量监控提醒" +# ) as dag: +# t1 = BashOperator( +# task_id='samples_notice', +# bash_command='python /opt/airflow/dags/repo/dags/jh/sample/samples_notice.py', +# ) +# t1 \ No newline at end of file diff --git a/dags/jh/sample/samples_notice.py b/dags/jh/sample/samples_notice.py index ceeb28f..038f05d 100644 --- a/dags/jh/sample/samples_notice.py +++ b/dags/jh/sample/samples_notice.py @@ -1,130 +1,130 @@ -from goojia_utils import ElasticSearchClient, dingutils, dateutils -import pandas as pd -import datetime -import logging - -log = logging.getLogger(__name__) - -# 获取当前日期和2个月前的日期,统计2个月样本数据 -now = datetime.datetime.today() -first_day = datetime.datetime(now.year, now.month, 1) -now_day = first_day.strftime('%Y%m%d') -last = dateutils.add_months(first_day, 0, -2, 0, 0) -last_day = last.strftime('%Y%m%d') - -log.info(f"系统当前时间 : {now}") - - -def proc(): - # 区域维度,统计2个月内样本数据量 - es206_client = ElasticSearchClient(host='192.168.0.206') - sql = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >=' + str(last_day) + ' and p_date<' + str(now_day) - - log.info(f"城市区域维度 sql : {sql}") - df = es206_client.scan_by_sql_to_data_frame(sql) - - df_region = pd.DataFrame(df.groupby(by=['city', 'region'])['_id'].count()).reset_index() - df_region = df_region.rename(columns={'_id': 'count'}) - df_region_res = df_region[df_region['count'] < 20] - - if df_region_res.empty == False: - text = '【房源样本数量自动检测】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情' - - # 计算2个月的日期,用来计算网站数据量减少百分比 - # 当前时间 - now = datetime.datetime.today() - # 获取当前月份的第一天 - first_day_of_month = datetime.datetime(now.year, now.month, 1) - # 获取1个月前的第一天 - last_month = first_day_of_month - datetime.timedelta(days=1) - last_day_of_month = last_month.replace(day=1) - # 获取2个月前的第一天 - last2_month = last_day_of_month - datetime.timedelta(days=1) - last2_day_of_month = last2_month.replace(day=1) - - strat_time = last_day_of_month.strftime('%Y%m%d') - end_time = first_day_of_month.strftime('%Y%m%d') - - strat_time2 = last2_day_of_month.strftime('%Y%m%d') - end_time2 = last_day_of_month.strftime('%Y%m%d') - - # 房源网站维度,按当月数量,和当月数量相比上月减少百分比 - sql1 = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >=' + strat_time + ' and p_date<=' + end_time - sql2 = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >=' + strat_time2 + ' and p_date<=' + end_time2 - log.info(f"房源网站维度-百分比 当月sql : {sql1}") - log.info(f"房源网站维度-百分比 上月sql : {sql2}") - df_root1 = es206_client.scan_by_sql_to_data_frame(sql1) - df_root2 = es206_client.scan_by_sql_to_data_frame(sql2) - - df_root_group1 = pd.DataFrame(df_root1.groupby(by=['root'])['_id'].count()).reset_index() - df_root_group2 = pd.DataFrame(df_root2.groupby(by=['root'])['_id'].count()).reset_index() - - df_root_group1 = df_root_group1.rename(columns={'_id': 'count'}) - df_root_group2 = df_root_group2.rename(columns={'_id': 'count'}) - - # 房源网站,百分比维度,减少30% - df_root_merge = pd.merge(df_root_group1, df_root_group2, how='left', on=['root']) - - count_new = 'count_' + strat_time[0:6] - count_old = 'count_' + strat_time2[0:6] - - df_root_merge = df_root_merge.rename(columns={'count_x': count_new, 'count_y': count_old}) - - df_root_res = df_root_merge.drop(df_root_merge[~((df_root_merge[count_new] < df_root_merge[count_old]) & ( - (df_root_merge[count_old] - df_root_merge[count_new]) / df_root_merge[count_old] > 0.3))].index) - - if df_root_res.empty == False: - if text == '': - text = '【房源样本数量自动检测】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情' - - # 房源网站,数量维度,前70个城市当月数量少于100 - - cities = pd.read_excel("/opt/airflow/dags/repo/dags/jh/para/估值模型测试城市模板.xlsx") - topCities = cities.head(70) - city_list = list(topCities['cityname']) - log.info(f"房源网站维度-数量 city_list : {city_list}") - - df_root_city_group1 = pd.DataFrame(df_root1.groupby(by=['root', 'city'])['_id'].count()).reset_index() - df_root_city_group1 = df_root_city_group1.rename(columns={'_id': 'count'}) - df_root_city_res = df_root_city_group1[ - (df_root_city_group1['city'].isin(city_list)) & (df_root_city_group1['count'] < 100)].reset_index() - - if df_root_city_res.empty == False: - if text == '': - text = '【房源样本数量自动检测】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情' - - path = "/home/airflow/data/jh/price" - file_name = f"{now_day}房源样本数量检测.xlsx" - air_file_name = path + '/' + file_name - return text, air_file_name, df_region_res, df_root_city_res, df_root_res, file_name, count_old, count_new - - -text, air_file_name, df_region_res, df_root_city_res, df_root_res, file_name, count_old, count_new = proc() - - -def up_to_nextcloud(): - """ - 上传统计信息至nextcloud - """ - from nextcloud import NextCloud - NEXTCLOUD_URL = "https://nextcloud.fangjia.com" - NEXTCLOUD_USERNAME = "shenjian@fangjia.com" - NEXTCLOUD_PASSWORD = "goojia123456" - nextcloud_file_name = '/共享/建行/' + file_name - nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD) - nxc.upload_file(air_file_name, nextcloud_file_name) - - -if text != '': - with pd.ExcelWriter(air_file_name) as writer: - if not df_region_res.empty: - df_region_res.to_excel(writer, sheet_name="房源样本数量检测-区域维度", columns=["city", "region", 'count'], index=False) - if not df_root_city_res.empty: - df_root_city_res.to_excel(writer, sheet_name="房源样本数量检测-房源网站维度(样本数)", columns=['root', 'city', 'count'], - index=False) - if not df_root_res.empty: - df_root_res.to_excel(writer, sheet_name="房源样本数量检测-房源网站维度(样本减少百分比)", columns=['root', count_old, count_new], - index=False) - up_to_nextcloud() - token = '1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736' - dingutils.send_text(text, token) +# from goojia_utils import ElasticSearchClient, dingutils, dateutils +# import pandas as pd +# import datetime +# import logging +# +# log = logging.getLogger(__name__) +# +# # 获取当前日期和2个月前的日期,统计2个月样本数据 +# now = datetime.datetime.today() +# first_day = datetime.datetime(now.year, now.month, 1) +# now_day = first_day.strftime('%Y%m%d') +# last = dateutils.add_months(first_day, 0, -2, 0, 0) +# last_day = last.strftime('%Y%m%d') +# +# log.info(f"系统当前时间 : {now}") +# +# +# def proc(): +# # 区域维度,统计2个月内样本数据量 +# es206_client = ElasticSearchClient(host='192.168.0.206') +# sql = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >=' + str(last_day) + ' and p_date<' + str(now_day) +# +# log.info(f"城市区域维度 sql : {sql}") +# df = es206_client.scan_by_sql_to_data_frame(sql) +# +# df_region = pd.DataFrame(df.groupby(by=['city', 'region'])['_id'].count()).reset_index() +# df_region = df_region.rename(columns={'_id': 'count'}) +# df_region_res = df_region[df_region['count'] < 20] +# +# if df_region_res.empty == False: +# text = '【房源样本数量自动检测】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情' +# +# # 计算2个月的日期,用来计算网站数据量减少百分比 +# # 当前时间 +# now = datetime.datetime.today() +# # 获取当前月份的第一天 +# first_day_of_month = datetime.datetime(now.year, now.month, 1) +# # 获取1个月前的第一天 +# last_month = first_day_of_month - datetime.timedelta(days=1) +# last_day_of_month = last_month.replace(day=1) +# # 获取2个月前的第一天 +# last2_month = last_day_of_month - datetime.timedelta(days=1) +# last2_day_of_month = last2_month.replace(day=1) +# +# strat_time = last_day_of_month.strftime('%Y%m%d') +# end_time = first_day_of_month.strftime('%Y%m%d') +# +# strat_time2 = last2_day_of_month.strftime('%Y%m%d') +# end_time2 = last_day_of_month.strftime('%Y%m%d') +# +# # 房源网站维度,按当月数量,和当月数量相比上月减少百分比 +# sql1 = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >=' + strat_time + ' and p_date<=' + end_time +# sql2 = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >=' + strat_time2 + ' and p_date<=' + end_time2 +# log.info(f"房源网站维度-百分比 当月sql : {sql1}") +# log.info(f"房源网站维度-百分比 上月sql : {sql2}") +# df_root1 = es206_client.scan_by_sql_to_data_frame(sql1) +# df_root2 = es206_client.scan_by_sql_to_data_frame(sql2) +# +# df_root_group1 = pd.DataFrame(df_root1.groupby(by=['root'])['_id'].count()).reset_index() +# df_root_group2 = pd.DataFrame(df_root2.groupby(by=['root'])['_id'].count()).reset_index() +# +# df_root_group1 = df_root_group1.rename(columns={'_id': 'count'}) +# df_root_group2 = df_root_group2.rename(columns={'_id': 'count'}) +# +# # 房源网站,百分比维度,减少30% +# df_root_merge = pd.merge(df_root_group1, df_root_group2, how='left', on=['root']) +# +# count_new = 'count_' + strat_time[0:6] +# count_old = 'count_' + strat_time2[0:6] +# +# df_root_merge = df_root_merge.rename(columns={'count_x': count_new, 'count_y': count_old}) +# +# df_root_res = df_root_merge.drop(df_root_merge[~((df_root_merge[count_new] < df_root_merge[count_old]) & ( +# (df_root_merge[count_old] - df_root_merge[count_new]) / df_root_merge[count_old] > 0.3))].index) +# +# if df_root_res.empty == False: +# if text == '': +# text = '【房源样本数量自动检测】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情' +# +# # 房源网站,数量维度,前70个城市当月数量少于100 +# +# cities = pd.read_excel("/opt/airflow/dags/repo/dags/jh/para/估值模型测试城市模板.xlsx") +# topCities = cities.head(70) +# city_list = list(topCities['cityname']) +# log.info(f"房源网站维度-数量 city_list : {city_list}") +# +# df_root_city_group1 = pd.DataFrame(df_root1.groupby(by=['root', 'city'])['_id'].count()).reset_index() +# df_root_city_group1 = df_root_city_group1.rename(columns={'_id': 'count'}) +# df_root_city_res = df_root_city_group1[ +# (df_root_city_group1['city'].isin(city_list)) & (df_root_city_group1['count'] < 100)].reset_index() +# +# if df_root_city_res.empty == False: +# if text == '': +# text = '【房源样本数量自动检测】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情' +# +# path = "/home/airflow/data/jh/price" +# file_name = f"{now_day}房源样本数量检测.xlsx" +# air_file_name = path + '/' + file_name +# return text, air_file_name, df_region_res, df_root_city_res, df_root_res, file_name, count_old, count_new +# +# +# text, air_file_name, df_region_res, df_root_city_res, df_root_res, file_name, count_old, count_new = proc() +# +# +# def up_to_nextcloud(): +# """ +# 上传统计信息至nextcloud +# """ +# from nextcloud import NextCloud +# NEXTCLOUD_URL = "https://nextcloud.fangjia.com" +# NEXTCLOUD_USERNAME = "shenjian@fangjia.com" +# NEXTCLOUD_PASSWORD = "goojia123456" +# nextcloud_file_name = '/共享/建行/' + file_name +# nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD) +# nxc.upload_file(air_file_name, nextcloud_file_name) +# +# +# if text != '': +# with pd.ExcelWriter(air_file_name) as writer: +# if not df_region_res.empty: +# df_region_res.to_excel(writer, sheet_name="房源样本数量检测-区域维度", columns=["city", "region", 'count'], index=False) +# if not df_root_city_res.empty: +# df_root_city_res.to_excel(writer, sheet_name="房源样本数量检测-房源网站维度(样本数)", columns=['root', 'city', 'count'], +# index=False) +# if not df_root_res.empty: +# df_root_res.to_excel(writer, sheet_name="房源样本数量检测-房源网站维度(样本减少百分比)", columns=['root', count_old, count_new], +# index=False) +# up_to_nextcloud() +# token = '1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736' +# dingutils.send_text(text, token) From 6e56ac4c020549dd6ec76c599cd24d850106034e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 14 Aug 2024 18:13:19 +0800 Subject: [PATCH 4/4] =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/jh/dag_samples_notice.py | 34 ++-- dags/jh/sample/samples_notice.py | 260 +++++++++++++++---------------- 2 files changed, 147 insertions(+), 147 deletions(-) diff --git a/dags/jh/dag_samples_notice.py b/dags/jh/dag_samples_notice.py index c00d38a..0f2a546 100644 --- a/dags/jh/dag_samples_notice.py +++ b/dags/jh/dag_samples_notice.py @@ -1,17 +1,17 @@ -# from airflow.operators.bash import BashOperator -# import pendulum -# from airflow import DAG -# -# with DAG( -# dag_id="samples_notice_dag", -# # 每月10日1点运行 -# schedule_interval="0 0 10 * *", -# start_date=pendulum.datetime(2023, 8, 1, tz="Asia/Shanghai"), -# catchup=True, -# description="样本数量监控提醒" -# ) as dag: -# t1 = BashOperator( -# task_id='samples_notice', -# bash_command='python /opt/airflow/dags/repo/dags/jh/sample/samples_notice.py', -# ) -# t1 \ No newline at end of file +from airflow.operators.bash import BashOperator +import pendulum +from airflow import DAG + +with DAG( + dag_id="samples_notice_dag", + # 每月10日1点运行 + schedule_interval="0 0 10 * *", + start_date=pendulum.datetime(2023, 8, 1, tz="Asia/Shanghai"), + catchup=True, + description="样本数量监控提醒" +) as dag: + t1 = BashOperator( + task_id='samples_notice', + bash_command='python /opt/airflow/dags/repo/dags/jh/sample/samples_notice.py', + ) + t1 \ No newline at end of file diff --git a/dags/jh/sample/samples_notice.py b/dags/jh/sample/samples_notice.py index 038f05d..ceeb28f 100644 --- a/dags/jh/sample/samples_notice.py +++ b/dags/jh/sample/samples_notice.py @@ -1,130 +1,130 @@ -# from goojia_utils import ElasticSearchClient, dingutils, dateutils -# import pandas as pd -# import datetime -# import logging -# -# log = logging.getLogger(__name__) -# -# # 获取当前日期和2个月前的日期,统计2个月样本数据 -# now = datetime.datetime.today() -# first_day = datetime.datetime(now.year, now.month, 1) -# now_day = first_day.strftime('%Y%m%d') -# last = dateutils.add_months(first_day, 0, -2, 0, 0) -# last_day = last.strftime('%Y%m%d') -# -# log.info(f"系统当前时间 : {now}") -# -# -# def proc(): -# # 区域维度,统计2个月内样本数据量 -# es206_client = ElasticSearchClient(host='192.168.0.206') -# sql = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >=' + str(last_day) + ' and p_date<' + str(now_day) -# -# log.info(f"城市区域维度 sql : {sql}") -# df = es206_client.scan_by_sql_to_data_frame(sql) -# -# df_region = pd.DataFrame(df.groupby(by=['city', 'region'])['_id'].count()).reset_index() -# df_region = df_region.rename(columns={'_id': 'count'}) -# df_region_res = df_region[df_region['count'] < 20] -# -# if df_region_res.empty == False: -# text = '【房源样本数量自动检测】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情' -# -# # 计算2个月的日期,用来计算网站数据量减少百分比 -# # 当前时间 -# now = datetime.datetime.today() -# # 获取当前月份的第一天 -# first_day_of_month = datetime.datetime(now.year, now.month, 1) -# # 获取1个月前的第一天 -# last_month = first_day_of_month - datetime.timedelta(days=1) -# last_day_of_month = last_month.replace(day=1) -# # 获取2个月前的第一天 -# last2_month = last_day_of_month - datetime.timedelta(days=1) -# last2_day_of_month = last2_month.replace(day=1) -# -# strat_time = last_day_of_month.strftime('%Y%m%d') -# end_time = first_day_of_month.strftime('%Y%m%d') -# -# strat_time2 = last2_day_of_month.strftime('%Y%m%d') -# end_time2 = last_day_of_month.strftime('%Y%m%d') -# -# # 房源网站维度,按当月数量,和当月数量相比上月减少百分比 -# sql1 = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >=' + strat_time + ' and p_date<=' + end_time -# sql2 = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >=' + strat_time2 + ' and p_date<=' + end_time2 -# log.info(f"房源网站维度-百分比 当月sql : {sql1}") -# log.info(f"房源网站维度-百分比 上月sql : {sql2}") -# df_root1 = es206_client.scan_by_sql_to_data_frame(sql1) -# df_root2 = es206_client.scan_by_sql_to_data_frame(sql2) -# -# df_root_group1 = pd.DataFrame(df_root1.groupby(by=['root'])['_id'].count()).reset_index() -# df_root_group2 = pd.DataFrame(df_root2.groupby(by=['root'])['_id'].count()).reset_index() -# -# df_root_group1 = df_root_group1.rename(columns={'_id': 'count'}) -# df_root_group2 = df_root_group2.rename(columns={'_id': 'count'}) -# -# # 房源网站,百分比维度,减少30% -# df_root_merge = pd.merge(df_root_group1, df_root_group2, how='left', on=['root']) -# -# count_new = 'count_' + strat_time[0:6] -# count_old = 'count_' + strat_time2[0:6] -# -# df_root_merge = df_root_merge.rename(columns={'count_x': count_new, 'count_y': count_old}) -# -# df_root_res = df_root_merge.drop(df_root_merge[~((df_root_merge[count_new] < df_root_merge[count_old]) & ( -# (df_root_merge[count_old] - df_root_merge[count_new]) / df_root_merge[count_old] > 0.3))].index) -# -# if df_root_res.empty == False: -# if text == '': -# text = '【房源样本数量自动检测】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情' -# -# # 房源网站,数量维度,前70个城市当月数量少于100 -# -# cities = pd.read_excel("/opt/airflow/dags/repo/dags/jh/para/估值模型测试城市模板.xlsx") -# topCities = cities.head(70) -# city_list = list(topCities['cityname']) -# log.info(f"房源网站维度-数量 city_list : {city_list}") -# -# df_root_city_group1 = pd.DataFrame(df_root1.groupby(by=['root', 'city'])['_id'].count()).reset_index() -# df_root_city_group1 = df_root_city_group1.rename(columns={'_id': 'count'}) -# df_root_city_res = df_root_city_group1[ -# (df_root_city_group1['city'].isin(city_list)) & (df_root_city_group1['count'] < 100)].reset_index() -# -# if df_root_city_res.empty == False: -# if text == '': -# text = '【房源样本数量自动检测】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情' -# -# path = "/home/airflow/data/jh/price" -# file_name = f"{now_day}房源样本数量检测.xlsx" -# air_file_name = path + '/' + file_name -# return text, air_file_name, df_region_res, df_root_city_res, df_root_res, file_name, count_old, count_new -# -# -# text, air_file_name, df_region_res, df_root_city_res, df_root_res, file_name, count_old, count_new = proc() -# -# -# def up_to_nextcloud(): -# """ -# 上传统计信息至nextcloud -# """ -# from nextcloud import NextCloud -# NEXTCLOUD_URL = "https://nextcloud.fangjia.com" -# NEXTCLOUD_USERNAME = "shenjian@fangjia.com" -# NEXTCLOUD_PASSWORD = "goojia123456" -# nextcloud_file_name = '/共享/建行/' + file_name -# nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD) -# nxc.upload_file(air_file_name, nextcloud_file_name) -# -# -# if text != '': -# with pd.ExcelWriter(air_file_name) as writer: -# if not df_region_res.empty: -# df_region_res.to_excel(writer, sheet_name="房源样本数量检测-区域维度", columns=["city", "region", 'count'], index=False) -# if not df_root_city_res.empty: -# df_root_city_res.to_excel(writer, sheet_name="房源样本数量检测-房源网站维度(样本数)", columns=['root', 'city', 'count'], -# index=False) -# if not df_root_res.empty: -# df_root_res.to_excel(writer, sheet_name="房源样本数量检测-房源网站维度(样本减少百分比)", columns=['root', count_old, count_new], -# index=False) -# up_to_nextcloud() -# token = '1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736' -# dingutils.send_text(text, token) +from goojia_utils import ElasticSearchClient, dingutils, dateutils +import pandas as pd +import datetime +import logging + +log = logging.getLogger(__name__) + +# 获取当前日期和2个月前的日期,统计2个月样本数据 +now = datetime.datetime.today() +first_day = datetime.datetime(now.year, now.month, 1) +now_day = first_day.strftime('%Y%m%d') +last = dateutils.add_months(first_day, 0, -2, 0, 0) +last_day = last.strftime('%Y%m%d') + +log.info(f"系统当前时间 : {now}") + + +def proc(): + # 区域维度,统计2个月内样本数据量 + es206_client = ElasticSearchClient(host='192.168.0.206') + sql = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >=' + str(last_day) + ' and p_date<' + str(now_day) + + log.info(f"城市区域维度 sql : {sql}") + df = es206_client.scan_by_sql_to_data_frame(sql) + + df_region = pd.DataFrame(df.groupby(by=['city', 'region'])['_id'].count()).reset_index() + df_region = df_region.rename(columns={'_id': 'count'}) + df_region_res = df_region[df_region['count'] < 20] + + if df_region_res.empty == False: + text = '【房源样本数量自动检测】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情' + + # 计算2个月的日期,用来计算网站数据量减少百分比 + # 当前时间 + now = datetime.datetime.today() + # 获取当前月份的第一天 + first_day_of_month = datetime.datetime(now.year, now.month, 1) + # 获取1个月前的第一天 + last_month = first_day_of_month - datetime.timedelta(days=1) + last_day_of_month = last_month.replace(day=1) + # 获取2个月前的第一天 + last2_month = last_day_of_month - datetime.timedelta(days=1) + last2_day_of_month = last2_month.replace(day=1) + + strat_time = last_day_of_month.strftime('%Y%m%d') + end_time = first_day_of_month.strftime('%Y%m%d') + + strat_time2 = last2_day_of_month.strftime('%Y%m%d') + end_time2 = last_day_of_month.strftime('%Y%m%d') + + # 房源网站维度,按当月数量,和当月数量相比上月减少百分比 + sql1 = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >=' + strat_time + ' and p_date<=' + end_time + sql2 = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >=' + strat_time2 + ' and p_date<=' + end_time2 + log.info(f"房源网站维度-百分比 当月sql : {sql1}") + log.info(f"房源网站维度-百分比 上月sql : {sql2}") + df_root1 = es206_client.scan_by_sql_to_data_frame(sql1) + df_root2 = es206_client.scan_by_sql_to_data_frame(sql2) + + df_root_group1 = pd.DataFrame(df_root1.groupby(by=['root'])['_id'].count()).reset_index() + df_root_group2 = pd.DataFrame(df_root2.groupby(by=['root'])['_id'].count()).reset_index() + + df_root_group1 = df_root_group1.rename(columns={'_id': 'count'}) + df_root_group2 = df_root_group2.rename(columns={'_id': 'count'}) + + # 房源网站,百分比维度,减少30% + df_root_merge = pd.merge(df_root_group1, df_root_group2, how='left', on=['root']) + + count_new = 'count_' + strat_time[0:6] + count_old = 'count_' + strat_time2[0:6] + + df_root_merge = df_root_merge.rename(columns={'count_x': count_new, 'count_y': count_old}) + + df_root_res = df_root_merge.drop(df_root_merge[~((df_root_merge[count_new] < df_root_merge[count_old]) & ( + (df_root_merge[count_old] - df_root_merge[count_new]) / df_root_merge[count_old] > 0.3))].index) + + if df_root_res.empty == False: + if text == '': + text = '【房源样本数量自动检测】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情' + + # 房源网站,数量维度,前70个城市当月数量少于100 + + cities = pd.read_excel("/opt/airflow/dags/repo/dags/jh/para/估值模型测试城市模板.xlsx") + topCities = cities.head(70) + city_list = list(topCities['cityname']) + log.info(f"房源网站维度-数量 city_list : {city_list}") + + df_root_city_group1 = pd.DataFrame(df_root1.groupby(by=['root', 'city'])['_id'].count()).reset_index() + df_root_city_group1 = df_root_city_group1.rename(columns={'_id': 'count'}) + df_root_city_res = df_root_city_group1[ + (df_root_city_group1['city'].isin(city_list)) & (df_root_city_group1['count'] < 100)].reset_index() + + if df_root_city_res.empty == False: + if text == '': + text = '【房源样本数量自动检测】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情' + + path = "/home/airflow/data/jh/price" + file_name = f"{now_day}房源样本数量检测.xlsx" + air_file_name = path + '/' + file_name + return text, air_file_name, df_region_res, df_root_city_res, df_root_res, file_name, count_old, count_new + + +text, air_file_name, df_region_res, df_root_city_res, df_root_res, file_name, count_old, count_new = proc() + + +def up_to_nextcloud(): + """ + 上传统计信息至nextcloud + """ + from nextcloud import NextCloud + NEXTCLOUD_URL = "https://nextcloud.fangjia.com" + NEXTCLOUD_USERNAME = "shenjian@fangjia.com" + NEXTCLOUD_PASSWORD = "goojia123456" + nextcloud_file_name = '/共享/建行/' + file_name + nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD) + nxc.upload_file(air_file_name, nextcloud_file_name) + + +if text != '': + with pd.ExcelWriter(air_file_name) as writer: + if not df_region_res.empty: + df_region_res.to_excel(writer, sheet_name="房源样本数量检测-区域维度", columns=["city", "region", 'count'], index=False) + if not df_root_city_res.empty: + df_root_city_res.to_excel(writer, sheet_name="房源样本数量检测-房源网站维度(样本数)", columns=['root', 'city', 'count'], + index=False) + if not df_root_res.empty: + df_root_res.to_excel(writer, sheet_name="房源样本数量检测-房源网站维度(样本减少百分比)", columns=['root', count_old, count_new], + index=False) + up_to_nextcloud() + token = '1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736' + dingutils.send_text(text, token)