From 82c3191983965136816af6aba11830a0ca69d106 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Thu, 10 Apr 2025 18:20:40 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E8=B4=A2=E5=8A=9B=E8=AF=84?= =?UTF-8?q?=E7=BA=A7=E8=A1=A5=E5=85=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../business/community/consumption.py | 60 ++++++++++++------- 1 file changed, 37 insertions(+), 23 deletions(-) diff --git a/dags/business/business/community/consumption.py b/dags/business/business/community/consumption.py index f392ac0..d9aaa60 100644 --- a/dags/business/business/community/consumption.py +++ b/dags/business/business/community/consumption.py @@ -1,6 +1,8 @@ """ 月消费潜力、评级计算 """ +import warnings + from goojia_utils import ElasticSearchClient, dateutils, nextcloudutils import pandas as pd import numpy as np @@ -14,7 +16,8 @@ from sklearn.mixture import GaussianMixture from sklearn.preprocessing import QuantileTransformer log = logging.getLogger(__name__) - +# 忽略警告 +warnings.filterwarnings("ignore", category=UserWarning) es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') index_name = "community_summary" path = "/opt/airflow/dags/repo/dags/business/business/community/data" @@ -31,16 +34,16 @@ reasonable_dict = reasonable_df.set_index('city')['是否合理'].to_dict() # Min-Max 归一化到 [1, 100] -def min_max_scale(df, column, new_min=1, new_max=100): +def min_max_scale(df, column, target_column, new_min=1, new_max=100): X_min = df[column].min() X_max = df[column].max() # 防止分母为零(如果所有值相等) if X_max == X_min: - df['consumption_score'] = new_min # 如果所有值相同,设为下界 + df[target_column] = new_min # 如果所有值相同,设为下界 else: - df['consumption_score'] = (new_min + (df[column] - X_min) * (new_max - new_min) / (X_max - X_min)) - df['consumption_score'] = df['consumption_score'].astype(int) + df[target_column] = (new_min + (df[column] - X_min) * (new_max - new_min) / (X_max - X_min)) + df[target_column] = df[target_column].astype(int) return df def calc_consumption_potential(city): @@ -58,7 +61,7 @@ def calc_consumption_potential(city): # 修正的物业费入模 community_summary_df['estate_charge_fix'].fillna(community_summary_df['estate_charge'], inplace=True) - community_summary_df = community_summary_df.dropna(how='any').copy() + #community_summary_df = community_summary_df.dropna(how='any').copy() if len(community_summary_df) == 0: log.warning(f"{city}无法评级,缺少因子") return [] @@ -140,7 +143,7 @@ def calc_consumption_rating(community_summary_df): 计算消费等级 """ df = community_summary_df[['_id', 'city', 'consumption_potential', 'total_price_score', 'estate_charge_score', 'relative_price_ranking_score', 'avg_cost_score']] - df = df.dropna(subset=['consumption_potential']) + #df = df.dropna(subset=['consumption_potential']) source = "normal_consumption_potential" target = "consumption_rating" actions = list() @@ -148,6 +151,7 @@ def calc_consumption_rating(community_summary_df): # 总评级 total_rate = 6 + score_df = pd.DataFrame() if len(df) < total_rate: # 对于样本低于5的直接排名 df[target] = df['consumption_potential'].rank(method='dense', ascending=False).astype(int) @@ -157,34 +161,44 @@ def calc_consumption_rating(community_summary_df): df["avg_cost_rating"] = df['avg_cost_score'].rank(method='dense', ascending=False).astype(int) else: # 极值设置 + score_df = gmm_train(total_rate, df.dropna(subset=['total_price_score']), "total_price_score", "total_price_rating") + score_df = min_max_scale(score_df, 'total_price_score', 'total_price_score_minmax') df = filter_min_max(df) - df = gmm_train(total_rate, df, source, target) + df = gmm_train(total_rate, df.dropna(subset=['consumption_potential']), source, target) # 新增子因子评级 - df = gmm_train(total_rate, df, "total_price_score", "total_price_rating") df = gmm_train(total_rate, df, "estate_charge_score", "estate_charge_rating") df = gmm_train(total_rate, df, "relative_price_ranking_score", "relative_price_rating") df = gmm_train(total_rate, df, "avg_cost_score", "avg_cost_rating") - # 填充缺失值 - df["consumption_rating"] = df["consumption_rating"].fillna(df["total_price_rating"]) - - # 应用归一化 - df = min_max_scale(df, 'consumption_potential') + # 应用归一化 + df = min_max_scale(df, 'consumption_potential', 'consumption_score') + if len(score_df) > 0: + # 补充财力评级 + df = pd.merge(score_df[['_id', 'total_price_rating', 'total_price_score_minmax']], df, how='left', on=['_id']) + # 填充缺失值 + df["consumption_rating"] = df["consumption_rating"].fillna(df["total_price_rating"]) + df["consumption_score"] = df["consumption_score"].fillna(df["total_price_score_minmax"]) # 批量更新 for index, item in df.iterrows(): row = dict() _id = str(item["_id"]) - row['consumption_rating'] = item['consumption_rating'] - row['consumption_score'] = item['consumption_score'] if pd.notna(item['consumption_score']) else None - row['consumption_potential'] = item['consumption_potential'] + if pd.notna(item['consumption_rating']): + row['consumption_rating'] = item['consumption_rating'] + if pd.notna(item['consumption_score']): + row['consumption_score'] = item['consumption_score'] + if pd.notna(item['consumption_potential']): + row['consumption_potential'] = item['consumption_potential'] row['m_date'] = dateutils.iso_format() - row['sub_factor'] = { - "total_price_rating": item['total_price_rating'], - "estate_charge_rating": item['estate_charge_rating'], - "relative_price_rating": item['relative_price_rating'], - "avg_cost_rating": item['avg_cost_rating'] - } + row['sub_factor'] = {} + if pd.notna(item['total_price_rating']): + row['sub_factor']['total_price_rating'] = item['total_price_rating'] + if pd.notna(item['estate_charge_rating']): + row['sub_factor']['estate_charge_rating'] = item['estate_charge_rating'] + if pd.notna(item['relative_price_rating']): + row['sub_factor']['relative_price_rating'] = item['relative_price_rating'] + if pd.notna(item['avg_cost_rating']): + row['sub_factor']['avg_cost_rating'] = item['avg_cost_rating'] action = {'_op_type': 'update', # 操作 index update create delete '_index': index_name, # index