From 504f4fb89cf610cd12150d3a109e975a87182c30 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Wed, 14 Dec 2022 15:39:59 +0800 Subject: [PATCH] =?UTF-8?q?=E8=B4=A2=E5=8A=9B=E8=AF=84=E7=BA=A7=E7=89=B9?= =?UTF-8?q?=E5=BE=81=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../business/community/consumption.py | 58 ++++++++++--------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/dags/business/business/community/consumption.py b/dags/business/business/community/consumption.py index dd3e21d..2b20c25 100644 --- a/dags/business/business/community/consumption.py +++ b/dags/business/business/community/consumption.py @@ -1,7 +1,7 @@ """ 月消费潜力、评级计算 """ -from goojia_utils import ElasticSearchClient, dateutils, nextcloudutils, dingutils, areautils +from goojia_utils import ElasticSearchClient, dateutils, nextcloudutils import pandas as pd import numpy as np import logging @@ -10,6 +10,7 @@ from elasticsearch import helpers import matplotlib.pyplot as plt from sklearn.mixture import GaussianMixture +from sklearn.preprocessing import QuantileTransformer log = logging.getLogger(__name__) @@ -54,13 +55,13 @@ def calc_consumption_potential(city): # 极值设置 community_summary_df = filter_min_max(city, community_summary_df).copy() - community_summary_df['total_price_mean'] = community_summary_df.groupby(by=['city'])['total_price'].transform( + community_summary_df['total_price_mean'] = community_summary_df.groupby(by=['city'])['normal_total_price'].transform( np.mean) - community_summary_df['total_price_std'] = community_summary_df.groupby(by=['city'])['total_price'].transform(np.std) + community_summary_df['total_price_std'] = community_summary_df.groupby(by=['city'])['normal_total_price'].transform(np.std) - community_summary_df['estate_charge_mean'] = community_summary_df.groupby(by=['city'])['estate_charge'].transform( + community_summary_df['estate_charge_mean'] = community_summary_df.groupby(by=['city'])['normal_estate_charge'].transform( np.mean) - community_summary_df['estate_charge_std'] = community_summary_df.groupby(by=['city'])['estate_charge'].transform( + community_summary_df['estate_charge_std'] = community_summary_df.groupby(by=['city'])['normal_estate_charge'].transform( np.std) community_summary_df['relative_price_ranking_mean'] = community_summary_df.groupby(by=['city'])[ @@ -68,8 +69,8 @@ def calc_consumption_potential(city): community_summary_df['relative_price_ranking_std'] = community_summary_df.groupby(by=['city'])[ 'relative_price_ranking'].transform(np.std) - community_summary_df['avg_cost_mean'] = community_summary_df.groupby(by=['city'])['avg_cost'].transform(np.mean) - community_summary_df['avg_cost_std'] = community_summary_df.groupby(by=['city'])['avg_cost'].transform(np.std) + community_summary_df['avg_cost_mean'] = community_summary_df.groupby(by=['city'])['normal_avg_cost'].transform(np.mean) + community_summary_df['avg_cost_std'] = community_summary_df.groupby(by=['city'])['normal_avg_cost'].transform(np.std) community_summary_df['total_price_score'] = 286.0134196 * ( community_summary_df['total_price'] - community_summary_df['total_price_mean']) / community_summary_df[ @@ -92,7 +93,7 @@ def calc_consumption_potential(city): def gmm_train(total_rate, consumption_df, source, target): # 高斯混合模型 - gmm = GaussianMixture(n_components=total_rate).fit(consumption_df[[source]]) + gmm = GaussianMixture(n_components=total_rate, covariance_type='tied', max_iter=200, n_init=10).fit(consumption_df[[source]]) consumption_df.loc[:, 'cluster'] = gmm.predict(consumption_df[[source]]) consumption_df.loc[:, 'price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean') consumption_df.loc[:, target] = consumption_df['price_mean_cluster'].rank(method='dense', ascending=False).astype(int) @@ -112,8 +113,10 @@ def calc_consumption_rating(community_summary_df): count = 0 if len(df) < 5: - return [] - df = gmm_train(5, df, source, target) + # 对于样本低于5的直接排名 + df[target] = df['consumption_potential'].rank(method='dense', ascending=False).astype(int) + else: + df = gmm_train(5, df, source, target) # 批量更新 for index, item in df.iterrows(): row = dict() @@ -165,23 +168,24 @@ def sync(city=None): def filter_min_max(city, community_summary_df): - """ - 过滤极值 - """ - total_price5 = np.percentile(a=community_summary_df['total_price'], q=2, axis=0) - total_price95 = np.percentile(a=community_summary_df['total_price'], q=98, axis=0) - community_summary_df.loc[community_summary_df['total_price'] > total_price95, 'total_price'] = total_price95 - community_summary_df.loc[community_summary_df['total_price'] < total_price5, 'total_price'] = total_price5 - - estate_charge5 = np.percentile(a=community_summary_df['estate_charge'], q=2, axis=0) - estate_charge95 = np.percentile(a=community_summary_df['estate_charge'], q=98, axis=0) - community_summary_df.loc[community_summary_df['estate_charge'] > estate_charge95, 'estate_charge'] = estate_charge95 - community_summary_df.loc[community_summary_df['estate_charge'] < estate_charge5, 'estate_charge'] = estate_charge5 - - avg_cost5 = np.percentile(a=community_summary_df['avg_cost'], q=2, axis=0) - avg_cost95 = np.percentile(a=community_summary_df['avg_cost'], q=98, axis=0) - community_summary_df.loc[community_summary_df['avg_cost'] > avg_cost95, 'avg_cost'] = avg_cost95 - community_summary_df.loc[community_summary_df['avg_cost'] < avg_cost5, 'avg_cost'] = avg_cost5 + normal_df = pd.DataFrame(QuantileTransformer(output_distribution="normal").fit_transform(community_summary_df[['total_price', 'estate_charge', 'avg_cost']]), columns=['total_price', 'estate_charge', 'avg_cost']) + community_summary_df['normal_total_price'] = normal_df['total_price'] + community_summary_df['normal_estate_charge'] = normal_df['estate_charge'] + community_summary_df['normal_avg_cost'] = normal_df['avg_cost'] + # total_price5 = np.percentile(a=community_summary_df['total_price'], q=2, axis=0) + # total_price95 = np.percentile(a=community_summary_df['total_price'], q=98, axis=0) + # community_summary_df.loc[community_summary_df['total_price'] > total_price95, 'total_price'] = total_price95 + # community_summary_df.loc[community_summary_df['total_price'] < total_price5, 'total_price'] = total_price5 + # + # estate_charge5 = np.percentile(a=community_summary_df['estate_charge'], q=2, axis=0) + # estate_charge95 = np.percentile(a=community_summary_df['estate_charge'], q=98, axis=0) + # community_summary_df.loc[community_summary_df['estate_charge'] > estate_charge95, 'estate_charge'] = estate_charge95 + # community_summary_df.loc[community_summary_df['estate_charge'] < estate_charge5, 'estate_charge'] = estate_charge5 + # + # avg_cost5 = np.percentile(a=community_summary_df['avg_cost'], q=2, axis=0) + # avg_cost95 = np.percentile(a=community_summary_df['avg_cost'], q=98, axis=0) + # community_summary_df.loc[community_summary_df['avg_cost'] > avg_cost95, 'avg_cost'] = avg_cost95 + # community_summary_df.loc[community_summary_df['avg_cost'] < avg_cost5, 'avg_cost'] = avg_cost5 #if city in factor_dict: # 户均总价按极值设置可靠性不高,当户均总价补充后,图中大于最大值的量非常多