From cc2a198a839893cf4e2d48948fe94aa5170ed835 Mon Sep 17 00:00:00 2001 From: wangwenyan Date: Wed, 14 Aug 2024 09:20:04 +0800 Subject: [PATCH 1/2] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=A5=BC=E7=9B=98?= =?UTF-8?q?=E8=AF=84=E7=BA=A7=E7=AE=97=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/jh/sample/samples_notice.py | 133 ++++++++++++++++--------------- 1 file changed, 69 insertions(+), 64 deletions(-) diff --git a/dags/jh/sample/samples_notice.py b/dags/jh/sample/samples_notice.py index d8080d2..69a1e8e 100644 --- a/dags/jh/sample/samples_notice.py +++ b/dags/jh/sample/samples_notice.py @@ -14,88 +14,93 @@ last_day = last.strftime('%Y%m%d') log.info(f"系统当前时间 : {now}") -# 区域维度,统计2个月内样本数据量 -es206_client = ElasticSearchClient(host='192.168.0.206') -sql = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >=' + str( - last_day) + ' and p_date<' + str(now_day) -log.info(f"城市区域维度 sql : {sql}") -df = es206_client.scan_by_sql_to_data_frame(sql) -df_region = pd.DataFrame(df.groupby(by=['city', 'region'])['_id'].count()).reset_index() -df_region = df_region.rename(columns={'_id': 'count'}) -df_region_res = df_region[df_region['count'] < 20] +def proc(): + # 区域维度,统计2个月内样本数据量 + es206_client = ElasticSearchClient(host='192.168.0.206') + sql = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >=' + str( + last_day) + ' and p_date<' + str(now_day) + log.info(f"城市区域维度 sql : {sql}") + df = es206_client.scan_by_sql_to_data_frame(sql) -if df_region_res.empty == False: - text = '【房源样本数量自动检测】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情' + df_region = pd.DataFrame(df.groupby(by=['city', 'region'])['_id'].count()).reset_index() + df_region = df_region.rename(columns={'_id': 'count'}) + df_region_res = df_region[df_region['count'] < 20] -# 计算2个月的日期,用来计算网站数据量减少百分比 -# 当前时间 -now = datetime.datetime.today() -# 获取当前月份的第一天 -first_day_of_month = datetime.datetime(now.year, now.month, 1) -# 获取1个月前的第一天 -last_month = first_day_of_month - datetime.timedelta(days=1) -last_day_of_month = last_month.replace(day=1) -# 获取2个月前的第一天 -last2_month = last_day_of_month - datetime.timedelta(days=1) -last2_day_of_month = last2_month.replace(day=1) + if df_region_res.empty == False: + text = '【房源样本数量自动检测】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情' -strat_time = last_day_of_month.strftime('%Y%m%d') -end_time = first_day_of_month.strftime('%Y%m%d') + # 计算2个月的日期,用来计算网站数据量减少百分比 + # 当前时间 + now = datetime.datetime.today() + # 获取当前月份的第一天 + first_day_of_month = datetime.datetime(now.year, now.month, 1) + # 获取1个月前的第一天 + last_month = first_day_of_month - datetime.timedelta(days=1) + last_day_of_month = last_month.replace(day=1) + # 获取2个月前的第一天 + last2_month = last_day_of_month - datetime.timedelta(days=1) + last2_day_of_month = last2_month.replace(day=1) -strat_time2 = last2_day_of_month.strftime('%Y%m%d') -end_time2 = last_day_of_month.strftime('%Y%m%d') + strat_time = last_day_of_month.strftime('%Y%m%d') + end_time = first_day_of_month.strftime('%Y%m%d') -# 房源网站维度,按当月数量,和当月数量相比上月减少百分比 -sql1 = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >=' + strat_time + ' and p_date<=' + end_time -sql2 = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >=' + strat_time2 + ' and p_date<=' + end_time2 -log.info(f"房源网站维度-百分比 当月sql : {sql1}") -log.info(f"房源网站维度-百分比 上月sql : {sql2}") -df_root1 = es206_client.scan_by_sql_to_data_frame(sql1) -df_root2 = es206_client.scan_by_sql_to_data_frame(sql2) + strat_time2 = last2_day_of_month.strftime('%Y%m%d') + end_time2 = last_day_of_month.strftime('%Y%m%d') -df_root_group1 = pd.DataFrame(df_root1.groupby(by=['root'])['_id'].count()).reset_index() -df_root_group2 = pd.DataFrame(df_root2.groupby(by=['root'])['_id'].count()).reset_index() + # 房源网站维度,按当月数量,和当月数量相比上月减少百分比 + sql1 = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >=' + strat_time + ' and p_date<=' + end_time + sql2 = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >=' + strat_time2 + ' and p_date<=' + end_time2 + log.info(f"房源网站维度-百分比 当月sql : {sql1}") + log.info(f"房源网站维度-百分比 上月sql : {sql2}") + df_root1 = es206_client.scan_by_sql_to_data_frame(sql1) + df_root2 = es206_client.scan_by_sql_to_data_frame(sql2) -df_root_group1 = df_root_group1.rename(columns={'_id': 'count'}) -df_root_group2 = df_root_group2.rename(columns={'_id': 'count'}) + df_root_group1 = pd.DataFrame(df_root1.groupby(by=['root'])['_id'].count()).reset_index() + df_root_group2 = pd.DataFrame(df_root2.groupby(by=['root'])['_id'].count()).reset_index() -# 房源网站,百分比维度,减少30% -df_root_merge = pd.merge(df_root_group1, df_root_group2, how='left', on=['root']) + df_root_group1 = df_root_group1.rename(columns={'_id': 'count'}) + df_root_group2 = df_root_group2.rename(columns={'_id': 'count'}) -count_new = 'count_' + strat_time[0:6] -count_old = 'count_' + strat_time2[0:6] + # 房源网站,百分比维度,减少30% + df_root_merge = pd.merge(df_root_group1, df_root_group2, how='left', on=['root']) -df_root_merge = df_root_merge.rename(columns={'count_x': count_new, 'count_y': count_old}) + count_new = 'count_' + strat_time[0:6] + count_old = 'count_' + strat_time2[0:6] -df_root_res = df_root_merge.drop(df_root_merge[~((df_root_merge[count_new] < df_root_merge[count_old]) & ( + df_root_merge = df_root_merge.rename(columns={'count_x': count_new, 'count_y': count_old}) + + df_root_res = df_root_merge.drop(df_root_merge[~((df_root_merge[count_new] < df_root_merge[count_old]) & ( (df_root_merge[count_old] - df_root_merge[count_new]) / df_root_merge[count_old] > 0.3))].index) -if df_root_res.empty == False: - if text == '': - text = '【房源样本数量自动检测】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情' + if df_root_res.empty == False: + if text == '': + text = '【房源样本数量自动检测】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情' -# 房源网站,数量维度,前70个城市当月数量少于100 + # 房源网站,数量维度,前70个城市当月数量少于100 -cities = pd.read_excel("/opt/airflow/dags/repo/dags/jh/para/估值模型测试城市模板.xlsx") -topCities = cities.head(70) -city_list = list(topCities['cityname']) -log.info(f"房源网站维度-数量 city_list : {city_list}") + cities = pd.read_excel("/opt/airflow/dags/repo/dags/jh/para/估值模型测试城市模板.xlsx") + topCities = cities.head(70) + city_list = list(topCities['cityname']) + log.info(f"房源网站维度-数量 city_list : {city_list}") -df_root_city_group1 = pd.DataFrame(df_root1.groupby(by=['root', 'city'])['_id'].count()).reset_index() -df_root_city_group1 = df_root_city_group1.rename(columns={'_id': 'count'}) -df_root_city_res = df_root_city_group1[ - (df_root_city_group1['city'].isin(city_list)) & (df_root_city_group1['count'] < 100)].reset_index() + df_root_city_group1 = pd.DataFrame(df_root1.groupby(by=['root', 'city'])['_id'].count()).reset_index() + df_root_city_group1 = df_root_city_group1.rename(columns={'_id': 'count'}) + df_root_city_res = df_root_city_group1[ + (df_root_city_group1['city'].isin(city_list)) & (df_root_city_group1['count'] < 100)].reset_index() -if df_root_city_res.empty == False: - if text == '': - text = '【房源样本数量自动检测】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情' + if df_root_city_res.empty == False: + if text == '': + text = '【房源样本数量自动检测】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情' -path = "/home/airflow/data/jh/price" -file_name = f"{now_day}房源样本数量检测.xlsx" -air_file_name = path + '/' + file_name + path = "/home/airflow/data/jh/price" + file_name = f"{now_day}房源样本数量检测.xlsx" + air_file_name = path + '/' + file_name + return text, air_file_name, df_region_res, df_root_city_res, df_root_res,file_name,count_old,count_new +text, air_file_name, df_region_res, df_root_city_res, df_root_res, file_name, count_old, count_new = proc() + def up_to_nextcloud(): """ 上传统计信息至nextcloud @@ -111,12 +116,12 @@ def up_to_nextcloud(): if text != '': with pd.ExcelWriter(air_file_name) as writer: - if df_region_res.empty == False: + if not df_region_res.empty: df_region_res.to_excel(writer, sheet_name="房源样本数量检测-区域维度", columns=["city", "region", 'count'], index=False) - if df_root_city_res.empty == False: + if not df_root_city_res.empty: df_root_city_res.to_excel(writer, sheet_name="房源样本数量检测-房源网站维度(样本数)", columns=['root', 'city', 'count'], index=False) - if df_root_res.empty == False: + if not df_root_res.empty: df_root_res.to_excel(writer, sheet_name="房源样本数量检测-房源网站维度(样本减少百分比)", columns=['root', count_old, count_new], index=False) up_to_nextcloud() From 7dc6a7eea6c1d9baa984d39399adbe0460ebd6d4 Mon Sep 17 00:00:00 2001 From: wangwenyan Date: Wed, 14 Aug 2024 09:21:05 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=A5=BC=E7=9B=98?= =?UTF-8?q?=E8=AF=84=E7=BA=A7=E7=AE=97=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/jh/sample/samples_notice.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dags/jh/sample/samples_notice.py b/dags/jh/sample/samples_notice.py index 69a1e8e..6074c3e 100644 --- a/dags/jh/sample/samples_notice.py +++ b/dags/jh/sample/samples_notice.py @@ -96,11 +96,12 @@ def proc(): path = "/home/airflow/data/jh/price" file_name = f"{now_day}房源样本数量检测.xlsx" air_file_name = path + '/' + file_name - return text, air_file_name, df_region_res, df_root_city_res, df_root_res,file_name,count_old,count_new + return text, air_file_name, df_region_res, df_root_city_res, df_root_res, file_name, count_old, count_new text, air_file_name, df_region_res, df_root_city_res, df_root_res, file_name, count_old, count_new = proc() + def up_to_nextcloud(): """ 上传统计信息至nextcloud