diff --git a/dags/business/business/community/consumption.py b/dags/business/business/community/consumption.py index b2115c8..68cdfee 100644 --- a/dags/business/business/community/consumption.py +++ b/dags/business/business/community/consumption.py @@ -12,6 +12,7 @@ from elasticsearch import helpers import matplotlib.pyplot as plt import sys +from sklearn.preprocessing import RobustScaler from sklearn.mixture import GaussianMixture from sklearn.preprocessing import QuantileTransformer @@ -31,6 +32,7 @@ nextcloud_aim_file_path = '共享/财力评级/财力评级图' reasonable_df = pd.read_excel(path + '/财力评级分布情况20221215.xlsx') reasonable_dict = reasonable_df.set_index('city')['是否合理'].to_dict() +scaler = RobustScaler() # Min-Max 归一化到 [1, 100] @@ -72,14 +74,17 @@ def calc_consumption_potential(city): # 户均总价计算时候改成万 community_summary_df['total_price'] = round(community_summary_df['total_price'] / 10000, 0) - # 噪音词处理 - community_summary_df['total_price_fixed'] = QuantileTransformer(output_distribution="normal").fit_transform( - community_summary_df[['total_price']]) - community_summary_df['avg_cost_fixed'] = QuantileTransformer(output_distribution="normal").fit_transform( - community_summary_df[['avg_cost']]) + community_summary_df['total_price_fixed'] = community_summary_df['total_price'] + community_summary_df['avg_cost_fixed'] = community_summary_df['avg_cost'] - community_summary_df['estate_charge_fixed'] = QuantileTransformer(output_distribution="normal").fit_transform( - community_summary_df[['estate_charge_fix']]) + community_summary_df['estate_charge_fixed'] = community_summary_df['estate_charge_fix'] + # community_summary_df['total_price_fixed'] = QuantileTransformer(output_distribution="normal").fit_transform( + # community_summary_df[['total_price']]) + # community_summary_df['avg_cost_fixed'] = QuantileTransformer(output_distribution="normal").fit_transform( + # community_summary_df[['avg_cost']]) + # + # community_summary_df['estate_charge_fixed'] = QuantileTransformer(output_distribution="normal").fit_transform( + # community_summary_df[['estate_charge_fix']]) community_summary_df['estate_charge_mean'] = community_summary_df.groupby(by=['city'])['estate_charge_fixed'].transform( np.mean) @@ -166,7 +171,8 @@ def calc_consumption_rating(community_summary_df): else: # 极值设置 score_df = gmm_train(total_rate, df.dropna(subset=['total_price_score']), "total_price_score", "total_price_rating") - score_df = min_max_scale(score_df, 'total_price_score', 'total_price_score_minmax') + score_df['robust_total_price_score'] = scaler.fit_transform(score_df[['total_price_score']]) + score_df = min_max_scale(score_df, 'robust_total_price_score', 'total_price_score_minmax') df = filter_min_max(df) if len(df.dropna(subset=['consumption_potential'])) >= total_rate: df = gmm_train(total_rate, df.dropna(subset=['consumption_potential']), source, target) @@ -176,7 +182,9 @@ def calc_consumption_rating(community_summary_df): df = gmm_train(total_rate, df, "avg_cost_score", "avg_cost_rating") # 应用归一化 - df = min_max_scale(df, 'consumption_potential', 'consumption_score') + df['robust_potential'] = scaler.fit_transform(df[['consumption_potential']]) + df = min_max_scale(df, 'robust_potential', 'consumption_score') # 再映射到 [1, 100] + #df = min_max_scale(df, 'consumption_potential', 'consumption_score') if len(score_df) > 0: # 补充财力评级 df = pd.merge(score_df[['_id', 'total_price_rating', 'total_price_score_minmax']], df, how='left', on=['_id']) @@ -289,40 +297,6 @@ def sync(city=None): def filter_min_max(community_summary_df): community_summary_df['normal_consumption_potential'] = QuantileTransformer(output_distribution="normal").fit_transform(community_summary_df[['consumption_potential']]) - # total_price5 = np.percentile(a=community_summary_df['total_price'], q=2, axis=0) - # total_price95 = np.percentile(a=community_summary_df['total_price'], q=98, axis=0) - # community_summary_df.loc[community_summary_df['total_price'] > total_price95, 'total_price'] = total_price95 - # community_summary_df.loc[community_summary_df['total_price'] < total_price5, 'total_price'] = total_price5 - # - # estate_charge5 = np.percentile(a=community_summary_df['estate_charge'], q=2, axis=0) - # estate_charge95 = np.percentile(a=community_summary_df['estate_charge'], q=98, axis=0) - # community_summary_df.loc[community_summary_df['estate_charge'] > estate_charge95, 'estate_charge'] = estate_charge95 - # community_summary_df.loc[community_summary_df['estate_charge'] < estate_charge5, 'estate_charge'] = estate_charge5 - # - # avg_cost5 = np.percentile(a=community_summary_df['avg_cost'], q=2, axis=0) - # avg_cost95 = np.percentile(a=community_summary_df['avg_cost'], q=98, axis=0) - # community_summary_df.loc[community_summary_df['avg_cost'] > avg_cost95, 'avg_cost'] = avg_cost95 - # community_summary_df.loc[community_summary_df['avg_cost'] < avg_cost5, 'avg_cost'] = avg_cost5 - - #if city in factor_dict: - # 户均总价按极值设置可靠性不高,当户均总价补充后,图中大于最大值的量非常多 - # community_summary_df.loc[ - # community_summary_df['total_price'] < factor_dict[city]['min_total_price'], 'total_price'] = \ - # factor_dict[city]['min_total_price'] - # community_summary_df.loc[ - # community_summary_df['total_price'] > factor_dict[city]['max_total_price'], 'total_price'] = \ - # factor_dict[city]['max_total_price'] - - # community_summary_df.loc[ - # community_summary_df['estate_charge'] < factor_dict[city]['min_estate_charge'], 'estate_charge'] = \ - # factor_dict[city]['min_estate_charge'] - # community_summary_df.loc[ - # community_summary_df['estate_charge'] > factor_dict[city]['max_estate_charge'], 'estate_charge'] = \ - # factor_dict[city]['max_estate_charge'] - # community_summary_df.loc[community_summary_df['avg_cost'] < factor_dict[city]['min_avg_cost'], 'avg_cost'] = \ - # factor_dict[city]['min_avg_cost'] - # community_summary_df.loc[community_summary_df['avg_cost'] > factor_dict[city]['max_avg_cost'], 'avg_cost'] = \ - # factor_dict[city]['max_avg_cost'] return community_summary_df diff --git a/dags/business/business/community/consumption_stat.py b/dags/business/business/community/consumption_stat.py index b63ade1..dd66d2a 100644 --- a/dags/business/business/community/consumption_stat.py +++ b/dags/business/business/community/consumption_stat.py @@ -146,7 +146,7 @@ def detail(city): '周边', '自有' ), - np.nan # 如果 estate_charge 为空,则返回 NaN + '' ) community_summary_df['credit'] = np.where(community_summary_df['total_price_diff'] == 0, '高', np.where(community_summary_df['total_price_diff'] == 1, '中', np.where(community_summary_df['total_price_diff'] >= 2, '低', '-')))