新增噪音词处理后评级
This commit is contained in:
@@ -10,6 +10,7 @@ import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
|
||||
from sklearn.mixture import GaussianMixture
|
||||
from sklearn.preprocessing import QuantileTransformer
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -36,22 +37,23 @@ for index, item in factor_df.iterrows():
|
||||
def gmm_train(total_rate, consumption_df, source, target):
|
||||
# 高斯混合模型
|
||||
gmm = GaussianMixture(n_components=total_rate).fit(consumption_df[[source]])
|
||||
consumption_df.loc[:, 'cluster'] = gmm.predict(consumption_df[[source]])
|
||||
consumption_df.loc[:, 'price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean')
|
||||
consumption_df.loc[:, target] = consumption_df['price_mean_cluster'].rank(method='dense', ascending=False).astype(int)
|
||||
consumption_df['cluster'] = gmm.predict(consumption_df[[source]])
|
||||
consumption_df['price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean')
|
||||
consumption_df[target] = consumption_df['price_mean_cluster'].rank(method='dense', ascending=False).astype(int)
|
||||
consumption_df = consumption_df.drop(columns=['cluster', 'price_mean_cluster'], axis=1)
|
||||
return consumption_df
|
||||
|
||||
|
||||
|
||||
def filter_min_max(city, community_summary_df):
|
||||
def filter_min_max(community_summary_df):
|
||||
"""
|
||||
过滤极值
|
||||
噪音词处理
|
||||
"""
|
||||
estate_charge5 = np.percentile(a=community_summary_df['estate_charge'], q=2, axis=0)
|
||||
estate_charge95 = np.percentile(a=community_summary_df['estate_charge'], q=98, axis=0)
|
||||
community_summary_df.loc[community_summary_df['estate_charge'] > estate_charge95, 'estate_charge'] = estate_charge95
|
||||
community_summary_df.loc[community_summary_df['estate_charge'] < estate_charge5, 'estate_charge'] = estate_charge5
|
||||
community_summary_df['QuantileTransformer.normal'] = QuantileTransformer(output_distribution="normal").fit_transform(community_summary_df[['estate_charge']])
|
||||
# estate_charge5 = np.percentile(a=community_summary_df['estate_charge'], q=1, axis=0)
|
||||
# estate_charge95 = np.percentile(a=community_summary_df['estate_charge'], q=99, axis=0)
|
||||
# community_summary_df.loc[community_summary_df['estate_charge'] > estate_charge95, 'estate_charge'] = estate_charge95
|
||||
# community_summary_df.loc[community_summary_df['estate_charge'] < estate_charge5, 'estate_charge'] = estate_charge5
|
||||
# if city in factor_dict:
|
||||
# community_summary_df.loc[
|
||||
# community_summary_df['estate_charge'] < factor_dict[city]['min_estate_charge'], 'estate_charge'] = \
|
||||
@@ -66,9 +68,8 @@ def calc_estate_charge_rating(community_summary_df):
|
||||
"""
|
||||
计算物业费等级
|
||||
"""
|
||||
df = community_summary_df[['_id', 'city', 'estate_charge']]
|
||||
df = df.dropna(subset=['estate_charge'])
|
||||
source = "estate_charge"
|
||||
df = community_summary_df[['_id', 'city', 'QuantileTransformer.normal', 'estate_charge']].copy()
|
||||
source = "QuantileTransformer.normal"
|
||||
target = "estate_charge_ranking"
|
||||
actions = list()
|
||||
count = 0
|
||||
@@ -110,8 +111,9 @@ def sync_by_city(city_now):
|
||||
if 'estate_charge' not in columns:
|
||||
log.warning(f"{city_now}无法评级,缺少因子")
|
||||
return
|
||||
community_summary_df = community_summary_df.dropna(how='any').copy()
|
||||
community_summary_df = filter_min_max(city_now, community_summary_df).copy()
|
||||
community_summary_df = community_summary_df.dropna(how='any').reset_index(drop=True).copy()
|
||||
# 噪音词处理
|
||||
community_summary_df = filter_min_max(community_summary_df)
|
||||
community_summary_df = calc_estate_charge_rating(community_summary_df)
|
||||
if len(community_summary_df) == 0:
|
||||
log.warning(f"{city_now}无法评级,数量低于5")
|
||||
|
||||
Reference in New Issue
Block a user