优化财力评级补充
This commit is contained in:
@@ -1,6 +1,8 @@
|
||||
"""
|
||||
月消费潜力、评级计算
|
||||
"""
|
||||
import warnings
|
||||
|
||||
from goojia_utils import ElasticSearchClient, dateutils, nextcloudutils
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
@@ -14,7 +16,8 @@ from sklearn.mixture import GaussianMixture
|
||||
from sklearn.preprocessing import QuantileTransformer
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# 忽略警告
|
||||
warnings.filterwarnings("ignore", category=UserWarning)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch')
|
||||
index_name = "community_summary"
|
||||
path = "/opt/airflow/dags/repo/dags/business/business/community/data"
|
||||
@@ -31,16 +34,16 @@ reasonable_dict = reasonable_df.set_index('city')['是否合理'].to_dict()
|
||||
|
||||
|
||||
# Min-Max 归一化到 [1, 100]
|
||||
def min_max_scale(df, column, new_min=1, new_max=100):
|
||||
def min_max_scale(df, column, target_column, new_min=1, new_max=100):
|
||||
X_min = df[column].min()
|
||||
X_max = df[column].max()
|
||||
|
||||
# 防止分母为零(如果所有值相等)
|
||||
if X_max == X_min:
|
||||
df['consumption_score'] = new_min # 如果所有值相同,设为下界
|
||||
df[target_column] = new_min # 如果所有值相同,设为下界
|
||||
else:
|
||||
df['consumption_score'] = (new_min + (df[column] - X_min) * (new_max - new_min) / (X_max - X_min))
|
||||
df['consumption_score'] = df['consumption_score'].astype(int)
|
||||
df[target_column] = (new_min + (df[column] - X_min) * (new_max - new_min) / (X_max - X_min))
|
||||
df[target_column] = df[target_column].astype(int)
|
||||
return df
|
||||
|
||||
def calc_consumption_potential(city):
|
||||
@@ -58,7 +61,7 @@ def calc_consumption_potential(city):
|
||||
# 修正的物业费入模
|
||||
community_summary_df['estate_charge_fix'].fillna(community_summary_df['estate_charge'], inplace=True)
|
||||
|
||||
community_summary_df = community_summary_df.dropna(how='any').copy()
|
||||
#community_summary_df = community_summary_df.dropna(how='any').copy()
|
||||
if len(community_summary_df) == 0:
|
||||
log.warning(f"{city}无法评级,缺少因子")
|
||||
return []
|
||||
@@ -140,7 +143,7 @@ def calc_consumption_rating(community_summary_df):
|
||||
计算消费等级
|
||||
"""
|
||||
df = community_summary_df[['_id', 'city', 'consumption_potential', 'total_price_score', 'estate_charge_score', 'relative_price_ranking_score', 'avg_cost_score']]
|
||||
df = df.dropna(subset=['consumption_potential'])
|
||||
#df = df.dropna(subset=['consumption_potential'])
|
||||
source = "normal_consumption_potential"
|
||||
target = "consumption_rating"
|
||||
actions = list()
|
||||
@@ -148,6 +151,7 @@ def calc_consumption_rating(community_summary_df):
|
||||
|
||||
# 总评级
|
||||
total_rate = 6
|
||||
score_df = pd.DataFrame()
|
||||
if len(df) < total_rate:
|
||||
# 对于样本低于5的直接排名
|
||||
df[target] = df['consumption_potential'].rank(method='dense', ascending=False).astype(int)
|
||||
@@ -157,34 +161,44 @@ def calc_consumption_rating(community_summary_df):
|
||||
df["avg_cost_rating"] = df['avg_cost_score'].rank(method='dense', ascending=False).astype(int)
|
||||
else:
|
||||
# 极值设置
|
||||
score_df = gmm_train(total_rate, df.dropna(subset=['total_price_score']), "total_price_score", "total_price_rating")
|
||||
score_df = min_max_scale(score_df, 'total_price_score', 'total_price_score_minmax')
|
||||
df = filter_min_max(df)
|
||||
df = gmm_train(total_rate, df, source, target)
|
||||
df = gmm_train(total_rate, df.dropna(subset=['consumption_potential']), source, target)
|
||||
# 新增子因子评级
|
||||
df = gmm_train(total_rate, df, "total_price_score", "total_price_rating")
|
||||
df = gmm_train(total_rate, df, "estate_charge_score", "estate_charge_rating")
|
||||
df = gmm_train(total_rate, df, "relative_price_ranking_score", "relative_price_rating")
|
||||
df = gmm_train(total_rate, df, "avg_cost_score", "avg_cost_rating")
|
||||
|
||||
# 填充缺失值
|
||||
df["consumption_rating"] = df["consumption_rating"].fillna(df["total_price_rating"])
|
||||
|
||||
# 应用归一化
|
||||
df = min_max_scale(df, 'consumption_potential')
|
||||
# 应用归一化
|
||||
df = min_max_scale(df, 'consumption_potential', 'consumption_score')
|
||||
if len(score_df) > 0:
|
||||
# 补充财力评级
|
||||
df = pd.merge(score_df[['_id', 'total_price_rating', 'total_price_score_minmax']], df, how='left', on=['_id'])
|
||||
# 填充缺失值
|
||||
df["consumption_rating"] = df["consumption_rating"].fillna(df["total_price_rating"])
|
||||
df["consumption_score"] = df["consumption_score"].fillna(df["total_price_score_minmax"])
|
||||
|
||||
# 批量更新
|
||||
for index, item in df.iterrows():
|
||||
row = dict()
|
||||
_id = str(item["_id"])
|
||||
row['consumption_rating'] = item['consumption_rating']
|
||||
row['consumption_score'] = item['consumption_score'] if pd.notna(item['consumption_score']) else None
|
||||
row['consumption_potential'] = item['consumption_potential']
|
||||
if pd.notna(item['consumption_rating']):
|
||||
row['consumption_rating'] = item['consumption_rating']
|
||||
if pd.notna(item['consumption_score']):
|
||||
row['consumption_score'] = item['consumption_score']
|
||||
if pd.notna(item['consumption_potential']):
|
||||
row['consumption_potential'] = item['consumption_potential']
|
||||
row['m_date'] = dateutils.iso_format()
|
||||
row['sub_factor'] = {
|
||||
"total_price_rating": item['total_price_rating'],
|
||||
"estate_charge_rating": item['estate_charge_rating'],
|
||||
"relative_price_rating": item['relative_price_rating'],
|
||||
"avg_cost_rating": item['avg_cost_rating']
|
||||
}
|
||||
row['sub_factor'] = {}
|
||||
if pd.notna(item['total_price_rating']):
|
||||
row['sub_factor']['total_price_rating'] = item['total_price_rating']
|
||||
if pd.notna(item['estate_charge_rating']):
|
||||
row['sub_factor']['estate_charge_rating'] = item['estate_charge_rating']
|
||||
if pd.notna(item['relative_price_rating']):
|
||||
row['sub_factor']['relative_price_rating'] = item['relative_price_rating']
|
||||
if pd.notna(item['avg_cost_rating']):
|
||||
row['sub_factor']['avg_cost_rating'] = item['avg_cost_rating']
|
||||
|
||||
action = {'_op_type': 'update', # 操作 index update create delete
|
||||
'_index': index_name, # index
|
||||
|
||||
Reference in New Issue
Block a user