修改房源网站统计规则,改为前70个热点城市当月数量少于100条提醒

This commit is contained in:
zhaojixiang@fangjia.com
2023-08-02 09:40:38 +08:00
parent f03bc87f6f
commit 780f4e469d
+31 -9
View File
@@ -1,6 +1,9 @@
from goojia_utils import ElasticSearchClient,dingutils,dateutils
import pandas as pd
import datetime
import logging
log = logging.getLogger(__name__)
# 获取当前日期和2个月前的日期,统计2个月样本数据
now = datetime.datetime.today()
@@ -8,9 +11,12 @@ now_day = now.strftime('%Y%m%d')
last = dateutils.add_months(now,0,-2,0,0)
last_day = last.strftime('%Y%m%d')
log.info(f"系统当前时间 : {now}")
# 区域维度,统计2个月内样本数据量
es206_client = ElasticSearchClient(host='192.168.0.206')
sql = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >='+str(last_day)+' and p_date<'+str(now_day)
log.info(f"城市区域维度 sql : {sql}")
df = es206_client.scan_by_sql_to_data_frame(sql)
df_region = pd.DataFrame(df.groupby(by=['city','region'])['_id'].count()).reset_index()
@@ -41,6 +47,8 @@ end_time2 = last_day_of_month.strftime('%Y%m%d')
# 房源网站维度,按当月数量,和当月数量相比上月减少百分比
sql1 = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >='+strat_time+' and p_date<='+end_time
sql2 = 'select city,region,p_date,cat,root from seawater where cat = \'sell\' and p_date >='+strat_time2+' and p_date<='+end_time2
log.info(f"房源网站维度-百分比 当月sql : {sql1}")
log.info(f"房源网站维度-百分比 上月sql : {sql2}")
df_root1 = es206_client.scan_by_sql_to_data_frame(sql1)
df_root2 = es206_client.scan_by_sql_to_data_frame(sql2)
@@ -51,18 +59,32 @@ df_root_group2 = pd.DataFrame(df_root2.groupby(by=['root'])['_id'].count()).rese
df_root_group1 = df_root_group1.rename(columns={'_id':'count'})
df_root_group2 = df_root_group2.rename(columns={'_id':'count'})
# 房源网站,数量维度,数量少于100
df_root_res1 = df_root_group1[df_root_group1['count']<100]
# 房源网站,百分比维度,减少30%
df_root_merge = pd.merge(df_root_group1,df_root_group2,how='left',on=['root'])
if df_root_res1.empty == False:
count_old = 'count_'+strat_time[0:6]
count_new = 'count_'+strat_time2[0:6]
df_root_merge = df_root_merge.rename(columns={'count_x':count_new,'count_y':count_old})
df_root_res = df_root_merge.drop(df_root_merge[~((df_root_merge['count_new']<df_root_merge['count_old']) & ((df_root_merge['count_old']-df_root_merge['count_new'])/df_root_merge['count_old']>0.3))].index)
if df_root_res.empty == False:
if text == '':
text = '【房源样本数量自动检测】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情'
# 房源网站,百分比维度,减少30%
df_root_merge = pd.merge(df_root_group1,df_root_group2,how='left',on=['root'])
df_root_res = df_root_merge.drop(df_root_merge[~((df_root_merge['count_x']<df_root_merge['count_y']) & ((df_root_merge['count_y']-df_root_merge['count_x'])/df_root_merge['count_y']>0.3))].index)
# 房源网站,数量维度,前70个城市当月数量少于100
if df_root_res.empty == False:
cities = pd.read_excel("/opt/airflow/dags/repo/dags/jh/para/估值模型测试城市模板.xlsx")
topCities = cities.head(70)
city_list = list(topCities['cityname'])
log.info(f"房源网站维度-数量 city_list : {city_list}")
df_root_city_group1 = pd.DataFrame(df_root1.groupby(by=['root','city'])['_id'].count()).reset_index()
df_root_city_group1 = df_root_city_group1.rename(columns={'_id':'count'})
df_root_city_res = df_root_city_group1[(df_root_city_group1['city'].isin(city_list)) & (df_root_city_group1['count']<100)].reset_index()
if df_root_city_res.empty == False:
if text == '':
text = '【房源样本数量自动检测】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情'
@@ -86,8 +108,8 @@ if text != '':
with pd.ExcelWriter(air_file_name) as writer:
if df_region_res.empty == False:
df_region_res.to_excel(writer, sheet_name="房源样本数量检测-区域维度", columns=["city","region",'count'], index=False)
if df_root_res1.empty == False:
df_root_res1.to_excel(writer, sheet_name="房源样本数量检测-房源网站维度(样本数)", columns=['root','count_x','count_y'], index=False)
if df_root_city_res.empty == False:
df_root_city_res.to_excel(writer, sheet_name="房源样本数量检测-房源网站维度(样本数)", columns=['root','city','count'], index=False)
if df_root_res.empty == False:
df_root_res.to_excel(writer, sheet_name="房源样本数量检测-房源网站维度(样本减少百分比)", columns=['root','count_x','count_y'], index=False)
up_to_nextcloud()