From 780f4e469ddc196ea8ad5144b4436f11c7f3429d Mon Sep 17 00:00:00 2001 From: "zhaojixiang@fangjia.com" Date: Wed, 2 Aug 2023 09:40:38 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=88=BF=E6=BA=90=E7=BD=91?= =?UTF-8?q?=E7=AB=99=E7=BB=9F=E8=AE=A1=E8=A7=84=E5=88=99,=E6=94=B9?= =?UTF-8?q?=E4=B8=BA=E5=89=8D70=E4=B8=AA=E7=83=AD=E7=82=B9=E5=9F=8E?= =?UTF-8?q?=E5=B8=82=E5=BD=93=E6=9C=88=E6=95=B0=E9=87=8F=E5=B0=91=E4=BA=8E?= =?UTF-8?q?100=E6=9D=A1=E6=8F=90=E9=86=92?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/jh/sample/samples_notice.py | 40 +++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/dags/jh/sample/samples_notice.py b/dags/jh/sample/samples_notice.py index cf7430e..40856c6 100644 --- a/dags/jh/sample/samples_notice.py +++ b/dags/jh/sample/samples_notice.py @@ -1,6 +1,9 @@ from goojia_utils import ElasticSearchClient,dingutils,dateutils import pandas as pd import datetime +import logging + +log = logging.getLogger(__name__) # 获取当前日期和2个月前的日期,统计2个月样本数据 now = datetime.datetime.today() @@ -8,9 +11,12 @@ now_day = now.strftime('%Y%m%d') last = dateutils.add_months(now,0,-2,0,0) last_day = last.strftime('%Y%m%d') +log.info(f"系统当前时间 : {now}") + # 区域维度,统计2个月内样本数据量 es206_client = ElasticSearchClient(host='192.168.0.206') sql = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >='+str(last_day)+' and p_date<'+str(now_day) +log.info(f"城市区域维度 sql : {sql}") df = es206_client.scan_by_sql_to_data_frame(sql) df_region = pd.DataFrame(df.groupby(by=['city','region'])['_id'].count()).reset_index() @@ -41,6 +47,8 @@ end_time2 = last_day_of_month.strftime('%Y%m%d') # 房源网站维度,按当月数量,和当月数量相比上月减少百分比 sql1 = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >='+strat_time+' and p_date<='+end_time sql2 = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >='+strat_time2+' and p_date<='+end_time2 +log.info(f"房源网站维度-百分比 当月sql : {sql1}") +log.info(f"房源网站维度-百分比 上月sql : {sql2}") df_root1 = es206_client.scan_by_sql_to_data_frame(sql1) df_root2 = es206_client.scan_by_sql_to_data_frame(sql2) @@ -51,18 +59,32 @@ df_root_group2 = pd.DataFrame(df_root2.groupby(by=['root'])['_id'].count()).rese df_root_group1 = df_root_group1.rename(columns={'_id':'count'}) df_root_group2 = df_root_group2.rename(columns={'_id':'count'}) -# 房源网站,数量维度,数量少于100 -df_root_res1 = df_root_group1[df_root_group1['count']<100] +# 房源网站,百分比维度,减少30% +df_root_merge = pd.merge(df_root_group1,df_root_group2,how='left',on=['root']) -if df_root_res1.empty == False: +count_old = 'count_'+strat_time[0:6] +count_new = 'count_'+strat_time2[0:6] + +df_root_merge = df_root_merge.rename(columns={'count_x':count_new,'count_y':count_old}) + +df_root_res = df_root_merge.drop(df_root_merge[~((df_root_merge['count_new']0.3))].index) + +if df_root_res.empty == False: if text == '': text = '【房源样本数量自动检测】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情' -# 房源网站,百分比维度,减少30% -df_root_merge = pd.merge(df_root_group1,df_root_group2,how='left',on=['root']) -df_root_res = df_root_merge.drop(df_root_merge[~((df_root_merge['count_x']0.3))].index) +# 房源网站,数量维度,前70个城市当月数量少于100 -if df_root_res.empty == False: +cities = pd.read_excel("/opt/airflow/dags/repo/dags/jh/para/估值模型测试城市模板.xlsx") +topCities = cities.head(70) +city_list = list(topCities['cityname']) +log.info(f"房源网站维度-数量 city_list : {city_list}") + +df_root_city_group1 = pd.DataFrame(df_root1.groupby(by=['root','city'])['_id'].count()).reset_index() +df_root_city_group1 = df_root_city_group1.rename(columns={'_id':'count'}) +df_root_city_res = df_root_city_group1[(df_root_city_group1['city'].isin(city_list)) & (df_root_city_group1['count']<100)].reset_index() + +if df_root_city_res.empty == False: if text == '': text = '【房源样本数量自动检测】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情' @@ -86,8 +108,8 @@ if text != '': with pd.ExcelWriter(air_file_name) as writer: if df_region_res.empty == False: df_region_res.to_excel(writer, sheet_name="房源样本数量检测-区域维度", columns=["city","region",'count'], index=False) - if df_root_res1.empty == False: - df_root_res1.to_excel(writer, sheet_name="房源样本数量检测-房源网站维度(样本数)", columns=['root','count_x','count_y'], index=False) + if df_root_city_res.empty == False: + df_root_city_res.to_excel(writer, sheet_name="房源样本数量检测-房源网站维度(样本数)", columns=['root','city','count'], index=False) if df_root_res.empty == False: df_root_res.to_excel(writer, sheet_name="房源样本数量检测-房源网站维度(样本减少百分比)", columns=['root','count_x','count_y'], index=False) up_to_nextcloud()