优化报错警告
This commit is contained in:
@@ -11,6 +11,7 @@ import os
|
|||||||
from elasticsearch import helpers
|
from elasticsearch import helpers
|
||||||
import matplotlib.pyplot as plt
|
import matplotlib.pyplot as plt
|
||||||
import sys
|
import sys
|
||||||
|
from scipy.stats import boxcox
|
||||||
|
|
||||||
from sklearn.preprocessing import RobustScaler
|
from sklearn.preprocessing import RobustScaler
|
||||||
from sklearn.mixture import GaussianMixture
|
from sklearn.mixture import GaussianMixture
|
||||||
@@ -91,7 +92,11 @@ def calc_consumption_potential(city):
|
|||||||
max_estate_charge_fixed = community_summary_df['estate_charge_fixed'].max()
|
max_estate_charge_fixed = community_summary_df['estate_charge_fixed'].max()
|
||||||
max_relative_price_ranking = community_summary_df['relative_price_ranking'].max()
|
max_relative_price_ranking = community_summary_df['relative_price_ranking'].max()
|
||||||
community_summary_df['total_price_fixed'] = 1 - np.log(community_summary_df['total_price_fixed'] + 1) / np.log(max_total_price_fixed + 1)
|
community_summary_df['total_price_fixed'] = 1 - np.log(community_summary_df['total_price_fixed'] + 1) / np.log(max_total_price_fixed + 1)
|
||||||
community_summary_df['avg_cost_fixed'] = 1 - np.log(community_summary_df['avg_cost_fixed'] + 1) / np.log(max_avg_cost_fixed + 1)
|
# 人均消费极大值处理
|
||||||
|
community_summary_df['avg_cost_fixed'] = np.where(community_summary_df['avg_cost_fixed'] > 250, 250, community_summary_df['avg_cost_fixed'])
|
||||||
|
|
||||||
|
community_summary_df['avg_cost_fixed'] = np.log1p(community_summary_df['avg_cost_fixed'])
|
||||||
|
#community_summary_df['avg_cost_fixed'] = 1 - np.log(community_summary_df['avg_cost_fixed'] + 1) / np.log(max_avg_cost_fixed + 1)
|
||||||
community_summary_df['estate_charge_fixed'] = 1 - np.log(community_summary_df['estate_charge_fixed'] + 1) / np.log(max_estate_charge_fixed + 1)
|
community_summary_df['estate_charge_fixed'] = 1 - np.log(community_summary_df['estate_charge_fixed'] + 1) / np.log(max_estate_charge_fixed + 1)
|
||||||
community_summary_df['relative_price_ranking'] = 1 - np.log(community_summary_df['relative_price_ranking'] + 1) / np.log(max_relative_price_ranking + 1)
|
community_summary_df['relative_price_ranking'] = 1 - np.log(community_summary_df['relative_price_ranking'] + 1) / np.log(max_relative_price_ranking + 1)
|
||||||
|
|
||||||
@@ -148,6 +153,7 @@ def gmm_train(total_rate, consumption_df, source, target):
|
|||||||
reg_covar=1e-6, # 正则化防止奇异矩阵
|
reg_covar=1e-6, # 正则化防止奇异矩阵
|
||||||
random_state=42
|
random_state=42
|
||||||
).fit(consumption_df[[source]])
|
).fit(consumption_df[[source]])
|
||||||
|
consumption_df = consumption_df.copy()
|
||||||
consumption_df.loc[:, 'cluster'] = gmm.predict(consumption_df[[source]])
|
consumption_df.loc[:, 'cluster'] = gmm.predict(consumption_df[[source]])
|
||||||
consumption_df.loc[:, 'price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean')
|
consumption_df.loc[:, 'price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean')
|
||||||
consumption_df.loc[:, target] = consumption_df['price_mean_cluster'].rank(method='dense', ascending=False).astype(int)
|
consumption_df.loc[:, target] = consumption_df['price_mean_cluster'].rank(method='dense', ascending=False).astype(int)
|
||||||
@@ -182,7 +188,8 @@ def calc_consumption_rating(community_summary_df):
|
|||||||
score_df = gmm_train(total_rate, df.dropna(subset=['total_price_score']), "total_price_score", "total_price_rating")
|
score_df = gmm_train(total_rate, df.dropna(subset=['total_price_score']), "total_price_score", "total_price_rating")
|
||||||
score_df['robust_total_price_score'] = scaler.fit_transform(score_df[['total_price_score']])
|
score_df['robust_total_price_score'] = scaler.fit_transform(score_df[['total_price_score']])
|
||||||
score_df = min_max_scale(score_df, 'robust_total_price_score', 'total_price_score_minmax')
|
score_df = min_max_scale(score_df, 'robust_total_price_score', 'total_price_score_minmax')
|
||||||
df = filter_min_max(df)
|
df = df.copy()
|
||||||
|
df.loc[:, 'normal_consumption_potential'] = QuantileTransformer(output_distribution="normal").fit_transform(df[['consumption_potential']])
|
||||||
if len(df.dropna(subset=['consumption_potential'])) >= total_rate:
|
if len(df.dropna(subset=['consumption_potential'])) >= total_rate:
|
||||||
df = gmm_train(total_rate, df.dropna(subset=['consumption_potential']), source, target)
|
df = gmm_train(total_rate, df.dropna(subset=['consumption_potential']), source, target)
|
||||||
# 新增子因子评级
|
# 新增子因子评级
|
||||||
@@ -253,6 +260,9 @@ def calc_consumption_rating(community_summary_df):
|
|||||||
count += len(actions)
|
count += len(actions)
|
||||||
actions.clear()
|
actions.clear()
|
||||||
log.info("评级完成,共更新: {}".format(count))
|
log.info("评级完成,共更新: {}".format(count))
|
||||||
|
#df['avg_cost_score_minmax'].hist(bins=100)
|
||||||
|
#df['consumption_score'].hist(bins=100)
|
||||||
|
#plt.show()
|
||||||
return pd.merge(community_summary_df, df, how='left', on=['_id', 'city', 'consumption_potential'])
|
return pd.merge(community_summary_df, df, how='left', on=['_id', 'city', 'consumption_potential'])
|
||||||
|
|
||||||
|
|
||||||
@@ -313,12 +323,6 @@ def sync(city=None):
|
|||||||
clear_rating(temp_df)
|
clear_rating(temp_df)
|
||||||
sync_by_city(city)
|
sync_by_city(city)
|
||||||
|
|
||||||
|
|
||||||
def filter_min_max(community_summary_df):
|
|
||||||
community_summary_df['normal_consumption_potential'] = QuantileTransformer(output_distribution="normal").fit_transform(community_summary_df[['consumption_potential']])
|
|
||||||
return community_summary_df
|
|
||||||
|
|
||||||
|
|
||||||
def save_png(community_city):
|
def save_png(community_city):
|
||||||
"""
|
"""
|
||||||
生成财力评级图表
|
生成财力评级图表
|
||||||
|
|||||||
Reference in New Issue
Block a user