优化楼盘评级-多进程处理

This commit is contained in:
2025-06-18 10:36:43 +08:00
parent cea2711e5f
commit 27319ed28d
+64 -61
View File
@@ -14,6 +14,8 @@ from sklearn.mixture import GaussianMixture
from sklearn.neighbors import BallTree
from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import StandardScaler
import multiprocessing
logger = logging.getLogger(__name__)
seaweed_client = MAIN_MONGO_CLIENT.get_collection("fangjia", "seaweed")
@@ -474,73 +476,74 @@ def load_data_by_city(city):
data['volume_price_ratio'] = data['volume_price_ratio'].fillna(data['volume_price_ratio'].mean())
return data
import multiprocessing
def process_city(city):
try:
logger.info(f"开始计算楼盘评级:{city}")
data = load_data_by_city(city)
data = comms_rating(data)
data["comm_id"] = data['_id']
db_fields = ["comm_id", "city", "region", "name", 'block', 'block_rating', "address", "sqmetre_rent", "yoy",
"rent_ro_price_ratio", "avg_price", "total_score", "community_quality_score",
"community_price_score", "location_advantage_score", "flowability_score", "living_facilities_score",
"education_score", "hospital_score", "rating", "community_quality_rating", "community_price_rating",
"location_advantage_rating", "flowability_rating", "living_facilities_rating", "education_rating",
"hospital_rating", "supermarket_num", "park_num", "catering_num", "mall_num", "kindergarten_num",
"key_primary_school_num", "ordinary_primary_school_num", "key_middle_school_num",
"ordinary_middle_school_num", "top_three_hospital_num", "non_3a_hospital_num", "build_year", "age",
"cubage_percent", "virescence_percent", "estate_charge", 'estate_type']
df_simple = data[db_fields].copy()
if "block" not in df_simple.columns:
df_simple['block'] = None
for factor_score_field in ['total_score', 'community_quality_score', 'community_price_score',
'location_advantage_score', 'flowability_score', 'living_facilities_score',
'education_score', 'hospital_score']:
key = "" if factor_score_field == "total_score" else f"_{factor_score_field.replace('_score', '')}"
df_simple[f'city{key}_rank'] = df_simple.groupby('city')[factor_score_field].rank(
method='dense', ascending=False).astype(int)
df_simple[f'city{key}_total_rank'] = df_simple[f'city{key}_rank'].max()
df_simple[f'region{key}_rank'] = df_simple.groupby(['city', 'region'])[factor_score_field].rank(
method='dense', ascending=False).astype(int)
df_simple[f'region{key}_total_rank'] = df_simple[f'region{key}_rank'].max()
df_simple[f'block{key}_rank'] = None
df_simple[f'block{key}_total_rank'] = None
if df_simple['block'].notna().any():
subset = df_simple[df_simple['block'].notna()]
df_simple.loc[subset.index, f'block{key}_rank'] = subset.groupby(
['city', 'region', 'block'])[factor_score_field].rank(method='dense', ascending=False).astype(int)
df_simple.loc[subset.index, f'block{key}_total_rank'] = df_simple.loc[subset.index][f'block{key}_rank'].max()
for score_factor in ['total_score', 'community_quality_score', 'community_price_score',
'location_advantage_score', 'flowability_score', 'living_facilities_score',
'education_score', 'hospital_score']:
df_simple[score_factor] = df_simple[score_factor].apply(lambda x: round(x, 2))
df_simple['rent_ro_price_ratio'] = df_simple['rent_ro_price_ratio'].apply(lambda x: round(x, 2))
df_simple['avg_price'] = df_simple['avg_price'].astype(int)
updater = MongoDataFrameUpdater(community_rate_client)
updater.bulk_insert_or_update_by_id(df_simple, id_fields=['comm_id'])
logger.info(f"楼盘评级计算结束:{city}")
except Exception as e:
logging.error(f"{city}楼盘评级计算失败;{e}")
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--city', type=str, default='all')
args = parser.parse_args()
if args.city == 'all':
cities = areautils.get_all_cities()
else:
cities = [city for city in args.city.split(',')]
cities = [city.strip() for city in args.city.split(',') if city.strip()]
db_fields = ["comm_id", "city", "region", "name", 'block', 'block_rating', "address", "sqmetre_rent", "yoy",
"rent_ro_price_ratio", "avg_price", "total_score", "community_quality_score",
"community_price_score",
"location_advantage_score", "flowability_score", "living_facilities_score", "education_score",
"hospital_score", "rating", "community_quality_rating", "community_price_rating",
"location_advantage_rating", "flowability_rating", "living_facilities_rating",
"education_rating", "hospital_rating", "supermarket_num", "park_num", "catering_num", "mall_num",
"kindergarten_num",
"key_primary_school_num", "ordinary_primary_school_num", "key_middle_school_num",
"ordinary_middle_school_num", "top_three_hospital_num", "non_3a_hospital_num", "build_year", "age",
"cubage_percent", "virescence_percent", "estate_charge", 'estate_type']
for city in cities:
try:
logger.info(f"开始计算楼盘评级:{city}")
data = load_data_by_city(city)
data = comms_rating(data)
data["comm_id"] = data['_id']
db_df = data[db_fields]
df_simple = db_df.copy()
if "block" not in df_simple.columns:
df_simple['block'] = None
for factor_score_field in ['total_score', 'community_quality_score', 'community_price_score',
'location_advantage_score', 'flowability_score', 'living_facilities_score',
'education_score', 'hospital_score']:
key = "_"
if factor_score_field != "total_score":
key = f"_{factor_score_field.replace('_score', '')}"
else:
key = ""
df_simple[f'city{key}_rank'] = df_simple.groupby('city')[factor_score_field].rank(method='dense',
ascending=False).astype(
int)
df_simple[f'city{key}_total_rank'] = df_simple[f'city{key}_rank'].max()
df_simple[f'region{key}_rank'] = df_simple.groupby(['city', 'region'])[factor_score_field].rank(
method='dense', ascending=False).astype(int)
df_simple[f'region{key}_total_rank'] = df_simple[f'region{key}_rank'].max()
# 处理bis为None的情况 - 只对非空bis值计算排名
df_simple[f'block{key}_total_rank'] = None
df_simple[f'block{key}_rank'] = None
# 对bis不为None的记录计算商圈排名
bis_not_null = df_simple['block'].notna()
if bis_not_null.any():
df_simple.loc[bis_not_null, f'block{key}_rank'] = \
df_simple.loc[bis_not_null].groupby(['city', 'region', 'block'])[factor_score_field].rank(
method='dense', ascending=False).astype(int)
df_simple.loc[bis_not_null, f'block{key}_total_rank'] = df_simple.loc[bis_not_null][
f'block{key}_rank'].max()
for score_factor in ['total_score', 'community_quality_score', 'community_price_score',
'location_advantage_score', 'flowability_score', 'living_facilities_score',
'education_score', 'hospital_score']:
df_simple[f'{score_factor}'] = df_simple[f'{score_factor}'].apply(lambda x: round(x, 2))
df_simple['rent_ro_price_ratio'] = df_simple['rent_ro_price_ratio'].apply(lambda x: round(x, 2))
df_simple['avg_price'] = df_simple['avg_price'].astype(int)
updater = MongoDataFrameUpdater(community_rate_client)
updater.bulk_insert_or_update_by_id(df_simple, id_fields=['comm_id'])
logger.info(f"楼盘评级计算结束:{city}")
except Exception as e:
logging.error(f"{city}楼盘评级计算失败;{e}")
with multiprocessing.Pool(processes=min(8, len(cities))) as pool:
pool.map(process_city, cities)