From 5d94eb7bd00ed80077a74d11f3afcf3193c7da68 Mon Sep 17 00:00:00 2001 From: wangwenyan Date: Mon, 15 Jul 2024 10:35:53 +0800 Subject: [PATCH] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=A5=BC=E7=9B=98=E8=AF=84?= =?UTF-8?q?=E7=BA=A7=E7=AE=97=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/jh/sample/samples_notice.py | 63 ++++++++++++++++---------------- dags/stat/community/rate.py | 2 +- 2 files changed, 32 insertions(+), 33 deletions(-) diff --git a/dags/jh/sample/samples_notice.py b/dags/jh/sample/samples_notice.py index da7d9f1..d8080d2 100644 --- a/dags/jh/sample/samples_notice.py +++ b/dags/jh/sample/samples_notice.py @@ -1,4 +1,4 @@ -from goojia_utils import ElasticSearchClient,dingutils,dateutils +from goojia_utils import ElasticSearchClient, dingutils, dateutils import pandas as pd import datetime import logging @@ -7,22 +7,23 @@ log = logging.getLogger(__name__) # 获取当前日期和2个月前的日期,统计2个月样本数据 now = datetime.datetime.today() -first_day = datetime.datetime(now.year,now.month, 1) +first_day = datetime.datetime(now.year, now.month, 1) now_day = first_day.strftime('%Y%m%d') -last = dateutils.add_months(first_day,0,-2,0,0) +last = dateutils.add_months(first_day, 0, -2, 0, 0) last_day = last.strftime('%Y%m%d') log.info(f"系统当前时间 : {now}") # 区域维度,统计2个月内样本数据量 es206_client = ElasticSearchClient(host='192.168.0.206') -sql = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >='+str(last_day)+' and p_date<'+str(now_day) +sql = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >=' + str( + last_day) + ' and p_date<' + str(now_day) log.info(f"城市区域维度 sql : {sql}") df = es206_client.scan_by_sql_to_data_frame(sql) -df_region = pd.DataFrame(df.groupby(by=['city','region'])['_id'].count()).reset_index() -df_region = df_region.rename(columns={'_id':'count'}) -df_region_res = df_region[df_region['count']<20] +df_region = pd.DataFrame(df.groupby(by=['city', 'region'])['_id'].count()).reset_index() +df_region = df_region.rename(columns={'_id': 'count'}) +df_region_res = df_region[df_region['count'] < 20] if df_region_res.empty == False: text = '【房源样本数量自动检测】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情' @@ -31,7 +32,7 @@ if df_region_res.empty == False: # 当前时间 now = datetime.datetime.today() # 获取当前月份的第一天 -first_day_of_month = datetime.datetime(now.year,now.month, 1) +first_day_of_month = datetime.datetime(now.year, now.month, 1) # 获取1个月前的第一天 last_month = first_day_of_month - datetime.timedelta(days=1) last_day_of_month = last_month.replace(day=1) @@ -46,29 +47,29 @@ strat_time2 = last2_day_of_month.strftime('%Y%m%d') end_time2 = last_day_of_month.strftime('%Y%m%d') # 房源网站维度,按当月数量,和当月数量相比上月减少百分比 -sql1 = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >='+strat_time+' and p_date<='+end_time -sql2 = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >='+strat_time2+' and p_date<='+end_time2 +sql1 = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >=' + strat_time + ' and p_date<=' + end_time +sql2 = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >=' + strat_time2 + ' and p_date<=' + end_time2 log.info(f"房源网站维度-百分比 当月sql : {sql1}") log.info(f"房源网站维度-百分比 上月sql : {sql2}") df_root1 = es206_client.scan_by_sql_to_data_frame(sql1) df_root2 = es206_client.scan_by_sql_to_data_frame(sql2) - df_root_group1 = pd.DataFrame(df_root1.groupby(by=['root'])['_id'].count()).reset_index() df_root_group2 = pd.DataFrame(df_root2.groupby(by=['root'])['_id'].count()).reset_index() -df_root_group1 = df_root_group1.rename(columns={'_id':'count'}) -df_root_group2 = df_root_group2.rename(columns={'_id':'count'}) +df_root_group1 = df_root_group1.rename(columns={'_id': 'count'}) +df_root_group2 = df_root_group2.rename(columns={'_id': 'count'}) # 房源网站,百分比维度,减少30% -df_root_merge = pd.merge(df_root_group1,df_root_group2,how='left',on=['root']) +df_root_merge = pd.merge(df_root_group1, df_root_group2, how='left', on=['root']) -count_new = 'count_'+strat_time[0:6] -count_old = 'count_'+strat_time2[0:6] +count_new = 'count_' + strat_time[0:6] +count_old = 'count_' + strat_time2[0:6] -df_root_merge = df_root_merge.rename(columns={'count_x':count_new,'count_y':count_old}) +df_root_merge = df_root_merge.rename(columns={'count_x': count_new, 'count_y': count_old}) -df_root_res = df_root_merge.drop(df_root_merge[~((df_root_merge[count_new]0.3))].index) +df_root_res = df_root_merge.drop(df_root_merge[~((df_root_merge[count_new] < df_root_merge[count_old]) & ( + (df_root_merge[count_old] - df_root_merge[count_new]) / df_root_merge[count_old] > 0.3))].index) if df_root_res.empty == False: if text == '': @@ -81,9 +82,10 @@ topCities = cities.head(70) city_list = list(topCities['cityname']) log.info(f"房源网站维度-数量 city_list : {city_list}") -df_root_city_group1 = pd.DataFrame(df_root1.groupby(by=['root','city'])['_id'].count()).reset_index() -df_root_city_group1 = df_root_city_group1.rename(columns={'_id':'count'}) -df_root_city_res = df_root_city_group1[(df_root_city_group1['city'].isin(city_list)) & (df_root_city_group1['count']<100)].reset_index() +df_root_city_group1 = pd.DataFrame(df_root1.groupby(by=['root', 'city'])['_id'].count()).reset_index() +df_root_city_group1 = df_root_city_group1.rename(columns={'_id': 'count'}) +df_root_city_res = df_root_city_group1[ + (df_root_city_group1['city'].isin(city_list)) & (df_root_city_group1['count'] < 100)].reset_index() if df_root_city_res.empty == False: if text == '': @@ -93,6 +95,7 @@ path = "/home/airflow/data/jh/price" file_name = f"{now_day}房源样本数量检测.xlsx" air_file_name = path + '/' + file_name + def up_to_nextcloud(): """ 上传统计信息至nextcloud @@ -105,21 +108,17 @@ def up_to_nextcloud(): nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD) nxc.upload_file(air_file_name, nextcloud_file_name) + if text != '': with pd.ExcelWriter(air_file_name) as writer: if df_region_res.empty == False: - df_region_res.to_excel(writer, sheet_name="房源样本数量检测-区域维度", columns=["city","region",'count'], index=False) + df_region_res.to_excel(writer, sheet_name="房源样本数量检测-区域维度", columns=["city", "region", 'count'], index=False) if df_root_city_res.empty == False: - df_root_city_res.to_excel(writer, sheet_name="房源样本数量检测-房源网站维度(样本数)", columns=['root','city','count'], index=False) + df_root_city_res.to_excel(writer, sheet_name="房源样本数量检测-房源网站维度(样本数)", columns=['root', 'city', 'count'], + index=False) if df_root_res.empty == False: - df_root_res.to_excel(writer, sheet_name="房源样本数量检测-房源网站维度(样本减少百分比)", columns=['root',count_old,count_new], index=False) + df_root_res.to_excel(writer, sheet_name="房源样本数量检测-房源网站维度(样本减少百分比)", columns=['root', count_old, count_new], + index=False) up_to_nextcloud() token = '1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736' - dingutils.send_text(text,token) - - - - - - - + dingutils.send_text(text, token) diff --git a/dags/stat/community/rate.py b/dags/stat/community/rate.py index 7286159..6f7c0ce 100644 --- a/dags/stat/community/rate.py +++ b/dags/stat/community/rate.py @@ -22,7 +22,7 @@ for server_area in server_area_conn.find({'display': {'$ne': '全国'}}, {"displ data = pd.DataFrame( district_stat_conn.find({"city": city, "weekend": weekend}, {"_id": 0, "region": 1, "district_name": 1, "avg_price": 1})) - if len(data['avg_price'].unique()) < 5: + if "avg_price" not in data or len(data['avg_price'].unique()) < 5: continue data['city'] = city logger.info(f"开始计算{city}楼盘评级 weekend:{dateutils.format_date(weekend, '%Y-%m-%d')}")