新增噪音词处理后评级

This commit is contained in:
shenjian@fangjia.com
2022-12-08 14:40:03 +08:00
parent 9020d31887
commit afb7ff0aae
@@ -10,6 +10,7 @@ import matplotlib.pyplot as plt
import numpy as np
from sklearn.mixture import GaussianMixture
from sklearn.preprocessing import QuantileTransformer
log = logging.getLogger(__name__)
@@ -36,22 +37,23 @@ for index, item in factor_df.iterrows():
def gmm_train(total_rate, consumption_df, source, target):
# 高斯混合模型
gmm = GaussianMixture(n_components=total_rate).fit(consumption_df[[source]])
consumption_df.loc[:, 'cluster'] = gmm.predict(consumption_df[[source]])
consumption_df.loc[:, 'price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean')
consumption_df.loc[:, target] = consumption_df['price_mean_cluster'].rank(method='dense', ascending=False).astype(int)
consumption_df['cluster'] = gmm.predict(consumption_df[[source]])
consumption_df['price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean')
consumption_df[target] = consumption_df['price_mean_cluster'].rank(method='dense', ascending=False).astype(int)
consumption_df = consumption_df.drop(columns=['cluster', 'price_mean_cluster'], axis=1)
return consumption_df
def filter_min_max(city, community_summary_df):
def filter_min_max(community_summary_df):
"""
过滤极值
噪音词处理
"""
estate_charge5 = np.percentile(a=community_summary_df['estate_charge'], q=2, axis=0)
estate_charge95 = np.percentile(a=community_summary_df['estate_charge'], q=98, axis=0)
community_summary_df.loc[community_summary_df['estate_charge'] > estate_charge95, 'estate_charge'] = estate_charge95
community_summary_df.loc[community_summary_df['estate_charge'] < estate_charge5, 'estate_charge'] = estate_charge5
community_summary_df['QuantileTransformer.normal'] = QuantileTransformer(output_distribution="normal").fit_transform(community_summary_df[['estate_charge']])
# estate_charge5 = np.percentile(a=community_summary_df['estate_charge'], q=1, axis=0)
# estate_charge95 = np.percentile(a=community_summary_df['estate_charge'], q=99, axis=0)
# community_summary_df.loc[community_summary_df['estate_charge'] > estate_charge95, 'estate_charge'] = estate_charge95
# community_summary_df.loc[community_summary_df['estate_charge'] < estate_charge5, 'estate_charge'] = estate_charge5
# if city in factor_dict:
# community_summary_df.loc[
# community_summary_df['estate_charge'] < factor_dict[city]['min_estate_charge'], 'estate_charge'] = \
@@ -66,9 +68,8 @@ def calc_estate_charge_rating(community_summary_df):
"""
计算物业费等级
"""
df = community_summary_df[['_id', 'city', 'estate_charge']]
df = df.dropna(subset=['estate_charge'])
source = "estate_charge"
df = community_summary_df[['_id', 'city', 'QuantileTransformer.normal', 'estate_charge']].copy()
source = "QuantileTransformer.normal"
target = "estate_charge_ranking"
actions = list()
count = 0
@@ -110,8 +111,9 @@ def sync_by_city(city_now):
if 'estate_charge' not in columns:
log.warning(f"{city_now}无法评级,缺少因子")
return
community_summary_df = community_summary_df.dropna(how='any').copy()
community_summary_df = filter_min_max(city_now, community_summary_df).copy()
community_summary_df = community_summary_df.dropna(how='any').reset_index(drop=True).copy()
# 噪音词处理
community_summary_df = filter_min_max(community_summary_df)
community_summary_df = calc_estate_charge_rating(community_summary_df)
if len(community_summary_df) == 0:
log.warning(f"{city_now}无法评级,数量低于5")