财力评级特征处理
This commit is contained in:
@@ -1,7 +1,7 @@
|
||||
"""
|
||||
月消费潜力、评级计算
|
||||
"""
|
||||
from goojia_utils import ElasticSearchClient, dateutils, nextcloudutils, dingutils, areautils
|
||||
from goojia_utils import ElasticSearchClient, dateutils, nextcloudutils
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
import logging
|
||||
@@ -10,6 +10,7 @@ from elasticsearch import helpers
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
from sklearn.mixture import GaussianMixture
|
||||
from sklearn.preprocessing import QuantileTransformer
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -54,13 +55,13 @@ def calc_consumption_potential(city):
|
||||
|
||||
# 极值设置
|
||||
community_summary_df = filter_min_max(city, community_summary_df).copy()
|
||||
community_summary_df['total_price_mean'] = community_summary_df.groupby(by=['city'])['total_price'].transform(
|
||||
community_summary_df['total_price_mean'] = community_summary_df.groupby(by=['city'])['normal_total_price'].transform(
|
||||
np.mean)
|
||||
community_summary_df['total_price_std'] = community_summary_df.groupby(by=['city'])['total_price'].transform(np.std)
|
||||
community_summary_df['total_price_std'] = community_summary_df.groupby(by=['city'])['normal_total_price'].transform(np.std)
|
||||
|
||||
community_summary_df['estate_charge_mean'] = community_summary_df.groupby(by=['city'])['estate_charge'].transform(
|
||||
community_summary_df['estate_charge_mean'] = community_summary_df.groupby(by=['city'])['normal_estate_charge'].transform(
|
||||
np.mean)
|
||||
community_summary_df['estate_charge_std'] = community_summary_df.groupby(by=['city'])['estate_charge'].transform(
|
||||
community_summary_df['estate_charge_std'] = community_summary_df.groupby(by=['city'])['normal_estate_charge'].transform(
|
||||
np.std)
|
||||
|
||||
community_summary_df['relative_price_ranking_mean'] = community_summary_df.groupby(by=['city'])[
|
||||
@@ -68,8 +69,8 @@ def calc_consumption_potential(city):
|
||||
community_summary_df['relative_price_ranking_std'] = community_summary_df.groupby(by=['city'])[
|
||||
'relative_price_ranking'].transform(np.std)
|
||||
|
||||
community_summary_df['avg_cost_mean'] = community_summary_df.groupby(by=['city'])['avg_cost'].transform(np.mean)
|
||||
community_summary_df['avg_cost_std'] = community_summary_df.groupby(by=['city'])['avg_cost'].transform(np.std)
|
||||
community_summary_df['avg_cost_mean'] = community_summary_df.groupby(by=['city'])['normal_avg_cost'].transform(np.mean)
|
||||
community_summary_df['avg_cost_std'] = community_summary_df.groupby(by=['city'])['normal_avg_cost'].transform(np.std)
|
||||
|
||||
community_summary_df['total_price_score'] = 286.0134196 * (
|
||||
community_summary_df['total_price'] - community_summary_df['total_price_mean']) / community_summary_df[
|
||||
@@ -92,7 +93,7 @@ def calc_consumption_potential(city):
|
||||
|
||||
def gmm_train(total_rate, consumption_df, source, target):
|
||||
# 高斯混合模型
|
||||
gmm = GaussianMixture(n_components=total_rate).fit(consumption_df[[source]])
|
||||
gmm = GaussianMixture(n_components=total_rate, covariance_type='tied', max_iter=200, n_init=10).fit(consumption_df[[source]])
|
||||
consumption_df.loc[:, 'cluster'] = gmm.predict(consumption_df[[source]])
|
||||
consumption_df.loc[:, 'price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean')
|
||||
consumption_df.loc[:, target] = consumption_df['price_mean_cluster'].rank(method='dense', ascending=False).astype(int)
|
||||
@@ -112,8 +113,10 @@ def calc_consumption_rating(community_summary_df):
|
||||
count = 0
|
||||
|
||||
if len(df) < 5:
|
||||
return []
|
||||
df = gmm_train(5, df, source, target)
|
||||
# 对于样本低于5的直接排名
|
||||
df[target] = df['consumption_potential'].rank(method='dense', ascending=False).astype(int)
|
||||
else:
|
||||
df = gmm_train(5, df, source, target)
|
||||
# 批量更新
|
||||
for index, item in df.iterrows():
|
||||
row = dict()
|
||||
@@ -165,23 +168,24 @@ def sync(city=None):
|
||||
|
||||
|
||||
def filter_min_max(city, community_summary_df):
|
||||
"""
|
||||
过滤极值
|
||||
"""
|
||||
total_price5 = np.percentile(a=community_summary_df['total_price'], q=2, axis=0)
|
||||
total_price95 = np.percentile(a=community_summary_df['total_price'], q=98, axis=0)
|
||||
community_summary_df.loc[community_summary_df['total_price'] > total_price95, 'total_price'] = total_price95
|
||||
community_summary_df.loc[community_summary_df['total_price'] < total_price5, 'total_price'] = total_price5
|
||||
|
||||
estate_charge5 = np.percentile(a=community_summary_df['estate_charge'], q=2, axis=0)
|
||||
estate_charge95 = np.percentile(a=community_summary_df['estate_charge'], q=98, axis=0)
|
||||
community_summary_df.loc[community_summary_df['estate_charge'] > estate_charge95, 'estate_charge'] = estate_charge95
|
||||
community_summary_df.loc[community_summary_df['estate_charge'] < estate_charge5, 'estate_charge'] = estate_charge5
|
||||
|
||||
avg_cost5 = np.percentile(a=community_summary_df['avg_cost'], q=2, axis=0)
|
||||
avg_cost95 = np.percentile(a=community_summary_df['avg_cost'], q=98, axis=0)
|
||||
community_summary_df.loc[community_summary_df['avg_cost'] > avg_cost95, 'avg_cost'] = avg_cost95
|
||||
community_summary_df.loc[community_summary_df['avg_cost'] < avg_cost5, 'avg_cost'] = avg_cost5
|
||||
normal_df = pd.DataFrame(QuantileTransformer(output_distribution="normal").fit_transform(community_summary_df[['total_price', 'estate_charge', 'avg_cost']]), columns=['total_price', 'estate_charge', 'avg_cost'])
|
||||
community_summary_df['normal_total_price'] = normal_df['total_price']
|
||||
community_summary_df['normal_estate_charge'] = normal_df['estate_charge']
|
||||
community_summary_df['normal_avg_cost'] = normal_df['avg_cost']
|
||||
# total_price5 = np.percentile(a=community_summary_df['total_price'], q=2, axis=0)
|
||||
# total_price95 = np.percentile(a=community_summary_df['total_price'], q=98, axis=0)
|
||||
# community_summary_df.loc[community_summary_df['total_price'] > total_price95, 'total_price'] = total_price95
|
||||
# community_summary_df.loc[community_summary_df['total_price'] < total_price5, 'total_price'] = total_price5
|
||||
#
|
||||
# estate_charge5 = np.percentile(a=community_summary_df['estate_charge'], q=2, axis=0)
|
||||
# estate_charge95 = np.percentile(a=community_summary_df['estate_charge'], q=98, axis=0)
|
||||
# community_summary_df.loc[community_summary_df['estate_charge'] > estate_charge95, 'estate_charge'] = estate_charge95
|
||||
# community_summary_df.loc[community_summary_df['estate_charge'] < estate_charge5, 'estate_charge'] = estate_charge5
|
||||
#
|
||||
# avg_cost5 = np.percentile(a=community_summary_df['avg_cost'], q=2, axis=0)
|
||||
# avg_cost95 = np.percentile(a=community_summary_df['avg_cost'], q=98, axis=0)
|
||||
# community_summary_df.loc[community_summary_df['avg_cost'] > avg_cost95, 'avg_cost'] = avg_cost95
|
||||
# community_summary_df.loc[community_summary_df['avg_cost'] < avg_cost5, 'avg_cost'] = avg_cost5
|
||||
|
||||
#if city in factor_dict:
|
||||
# 户均总价按极值设置可靠性不高,当户均总价补充后,图中大于最大值的量非常多
|
||||
|
||||
Reference in New Issue
Block a user