diff --git a/dags/business/business/community/estate_charge_ranking.py b/dags/business/business/community/estate_charge_ranking.py index 84eb3ef..682183f 100644 --- a/dags/business/business/community/estate_charge_ranking.py +++ b/dags/business/business/community/estate_charge_ranking.py @@ -10,6 +10,7 @@ import matplotlib.pyplot as plt import numpy as np from sklearn.mixture import GaussianMixture +from sklearn.preprocessing import QuantileTransformer log = logging.getLogger(__name__) @@ -36,22 +37,23 @@ for index, item in factor_df.iterrows(): def gmm_train(total_rate, consumption_df, source, target): # 高斯混合模型 gmm = GaussianMixture(n_components=total_rate).fit(consumption_df[[source]]) - consumption_df.loc[:, 'cluster'] = gmm.predict(consumption_df[[source]]) - consumption_df.loc[:, 'price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean') - consumption_df.loc[:, target] = consumption_df['price_mean_cluster'].rank(method='dense', ascending=False).astype(int) + consumption_df['cluster'] = gmm.predict(consumption_df[[source]]) + consumption_df['price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean') + consumption_df[target] = consumption_df['price_mean_cluster'].rank(method='dense', ascending=False).astype(int) consumption_df = consumption_df.drop(columns=['cluster', 'price_mean_cluster'], axis=1) return consumption_df -def filter_min_max(city, community_summary_df): +def filter_min_max(community_summary_df): """ - 过滤极值 + 噪音词处理 """ - estate_charge5 = np.percentile(a=community_summary_df['estate_charge'], q=2, axis=0) - estate_charge95 = np.percentile(a=community_summary_df['estate_charge'], q=98, axis=0) - community_summary_df.loc[community_summary_df['estate_charge'] > estate_charge95, 'estate_charge'] = estate_charge95 - community_summary_df.loc[community_summary_df['estate_charge'] < estate_charge5, 'estate_charge'] = estate_charge5 + community_summary_df['QuantileTransformer.normal'] = QuantileTransformer(output_distribution="normal").fit_transform(community_summary_df[['estate_charge']]) + # estate_charge5 = np.percentile(a=community_summary_df['estate_charge'], q=1, axis=0) + # estate_charge95 = np.percentile(a=community_summary_df['estate_charge'], q=99, axis=0) + # community_summary_df.loc[community_summary_df['estate_charge'] > estate_charge95, 'estate_charge'] = estate_charge95 + # community_summary_df.loc[community_summary_df['estate_charge'] < estate_charge5, 'estate_charge'] = estate_charge5 # if city in factor_dict: # community_summary_df.loc[ # community_summary_df['estate_charge'] < factor_dict[city]['min_estate_charge'], 'estate_charge'] = \ @@ -66,9 +68,8 @@ def calc_estate_charge_rating(community_summary_df): """ 计算物业费等级 """ - df = community_summary_df[['_id', 'city', 'estate_charge']] - df = df.dropna(subset=['estate_charge']) - source = "estate_charge" + df = community_summary_df[['_id', 'city', 'QuantileTransformer.normal', 'estate_charge']].copy() + source = "QuantileTransformer.normal" target = "estate_charge_ranking" actions = list() count = 0 @@ -110,8 +111,9 @@ def sync_by_city(city_now): if 'estate_charge' not in columns: log.warning(f"{city_now}无法评级,缺少因子") return - community_summary_df = community_summary_df.dropna(how='any').copy() - community_summary_df = filter_min_max(city_now, community_summary_df).copy() + community_summary_df = community_summary_df.dropna(how='any').reset_index(drop=True).copy() + # 噪音词处理 + community_summary_df = filter_min_max(community_summary_df) community_summary_df = calc_estate_charge_rating(community_summary_df) if len(community_summary_df) == 0: log.warning(f"{city_now}无法评级,数量低于5")