财力评级特征处理

This commit is contained in:
shenjian@fangjia.com
2022-12-14 15:39:59 +08:00
parent 452334f320
commit 504f4fb89c
+31 -27
View File
@@ -1,7 +1,7 @@
"""
月消费潜力、评级计算
"""
from goojia_utils import ElasticSearchClient, dateutils, nextcloudutils, dingutils, areautils
from goojia_utils import ElasticSearchClient, dateutils, nextcloudutils
import pandas as pd
import numpy as np
import logging
@@ -10,6 +10,7 @@ from elasticsearch import helpers
import matplotlib.pyplot as plt
from sklearn.mixture import GaussianMixture
from sklearn.preprocessing import QuantileTransformer
log = logging.getLogger(__name__)
@@ -54,13 +55,13 @@ def calc_consumption_potential(city):
# 极值设置
community_summary_df = filter_min_max(city, community_summary_df).copy()
community_summary_df['total_price_mean'] = community_summary_df.groupby(by=['city'])['total_price'].transform(
community_summary_df['total_price_mean'] = community_summary_df.groupby(by=['city'])['normal_total_price'].transform(
np.mean)
community_summary_df['total_price_std'] = community_summary_df.groupby(by=['city'])['total_price'].transform(np.std)
community_summary_df['total_price_std'] = community_summary_df.groupby(by=['city'])['normal_total_price'].transform(np.std)
community_summary_df['estate_charge_mean'] = community_summary_df.groupby(by=['city'])['estate_charge'].transform(
community_summary_df['estate_charge_mean'] = community_summary_df.groupby(by=['city'])['normal_estate_charge'].transform(
np.mean)
community_summary_df['estate_charge_std'] = community_summary_df.groupby(by=['city'])['estate_charge'].transform(
community_summary_df['estate_charge_std'] = community_summary_df.groupby(by=['city'])['normal_estate_charge'].transform(
np.std)
community_summary_df['relative_price_ranking_mean'] = community_summary_df.groupby(by=['city'])[
@@ -68,8 +69,8 @@ def calc_consumption_potential(city):
community_summary_df['relative_price_ranking_std'] = community_summary_df.groupby(by=['city'])[
'relative_price_ranking'].transform(np.std)
community_summary_df['avg_cost_mean'] = community_summary_df.groupby(by=['city'])['avg_cost'].transform(np.mean)
community_summary_df['avg_cost_std'] = community_summary_df.groupby(by=['city'])['avg_cost'].transform(np.std)
community_summary_df['avg_cost_mean'] = community_summary_df.groupby(by=['city'])['normal_avg_cost'].transform(np.mean)
community_summary_df['avg_cost_std'] = community_summary_df.groupby(by=['city'])['normal_avg_cost'].transform(np.std)
community_summary_df['total_price_score'] = 286.0134196 * (
community_summary_df['total_price'] - community_summary_df['total_price_mean']) / community_summary_df[
@@ -92,7 +93,7 @@ def calc_consumption_potential(city):
def gmm_train(total_rate, consumption_df, source, target):
# 高斯混合模型
gmm = GaussianMixture(n_components=total_rate).fit(consumption_df[[source]])
gmm = GaussianMixture(n_components=total_rate, covariance_type='tied', max_iter=200, n_init=10).fit(consumption_df[[source]])
consumption_df.loc[:, 'cluster'] = gmm.predict(consumption_df[[source]])
consumption_df.loc[:, 'price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean')
consumption_df.loc[:, target] = consumption_df['price_mean_cluster'].rank(method='dense', ascending=False).astype(int)
@@ -112,8 +113,10 @@ def calc_consumption_rating(community_summary_df):
count = 0
if len(df) < 5:
return []
df = gmm_train(5, df, source, target)
# 对于样本低于5的直接排名
df[target] = df['consumption_potential'].rank(method='dense', ascending=False).astype(int)
else:
df = gmm_train(5, df, source, target)
# 批量更新
for index, item in df.iterrows():
row = dict()
@@ -165,23 +168,24 @@ def sync(city=None):
def filter_min_max(city, community_summary_df):
"""
过滤极值
"""
total_price5 = np.percentile(a=community_summary_df['total_price'], q=2, axis=0)
total_price95 = np.percentile(a=community_summary_df['total_price'], q=98, axis=0)
community_summary_df.loc[community_summary_df['total_price'] > total_price95, 'total_price'] = total_price95
community_summary_df.loc[community_summary_df['total_price'] < total_price5, 'total_price'] = total_price5
estate_charge5 = np.percentile(a=community_summary_df['estate_charge'], q=2, axis=0)
estate_charge95 = np.percentile(a=community_summary_df['estate_charge'], q=98, axis=0)
community_summary_df.loc[community_summary_df['estate_charge'] > estate_charge95, 'estate_charge'] = estate_charge95
community_summary_df.loc[community_summary_df['estate_charge'] < estate_charge5, 'estate_charge'] = estate_charge5
avg_cost5 = np.percentile(a=community_summary_df['avg_cost'], q=2, axis=0)
avg_cost95 = np.percentile(a=community_summary_df['avg_cost'], q=98, axis=0)
community_summary_df.loc[community_summary_df['avg_cost'] > avg_cost95, 'avg_cost'] = avg_cost95
community_summary_df.loc[community_summary_df['avg_cost'] < avg_cost5, 'avg_cost'] = avg_cost5
normal_df = pd.DataFrame(QuantileTransformer(output_distribution="normal").fit_transform(community_summary_df[['total_price', 'estate_charge', 'avg_cost']]), columns=['total_price', 'estate_charge', 'avg_cost'])
community_summary_df['normal_total_price'] = normal_df['total_price']
community_summary_df['normal_estate_charge'] = normal_df['estate_charge']
community_summary_df['normal_avg_cost'] = normal_df['avg_cost']
# total_price5 = np.percentile(a=community_summary_df['total_price'], q=2, axis=0)
# total_price95 = np.percentile(a=community_summary_df['total_price'], q=98, axis=0)
# community_summary_df.loc[community_summary_df['total_price'] > total_price95, 'total_price'] = total_price95
# community_summary_df.loc[community_summary_df['total_price'] < total_price5, 'total_price'] = total_price5
#
# estate_charge5 = np.percentile(a=community_summary_df['estate_charge'], q=2, axis=0)
# estate_charge95 = np.percentile(a=community_summary_df['estate_charge'], q=98, axis=0)
# community_summary_df.loc[community_summary_df['estate_charge'] > estate_charge95, 'estate_charge'] = estate_charge95
# community_summary_df.loc[community_summary_df['estate_charge'] < estate_charge5, 'estate_charge'] = estate_charge5
#
# avg_cost5 = np.percentile(a=community_summary_df['avg_cost'], q=2, axis=0)
# avg_cost95 = np.percentile(a=community_summary_df['avg_cost'], q=98, axis=0)
# community_summary_df.loc[community_summary_df['avg_cost'] > avg_cost95, 'avg_cost'] = avg_cost95
# community_summary_df.loc[community_summary_df['avg_cost'] < avg_cost5, 'avg_cost'] = avg_cost5
#if city in factor_dict:
# 户均总价按极值设置可靠性不高,当户均总价补充后,图中大于最大值的量非常多