From 27319ed28d305503bb2124174b61beabe05f8215 Mon Sep 17 00:00:00 2001 From: wangwenyan Date: Wed, 18 Jun 2025 10:36:43 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=A5=BC=E7=9B=98=E8=AF=84?= =?UTF-8?q?=E7=BA=A7-=E5=A4=9A=E8=BF=9B=E7=A8=8B=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/stat/community/community_rate.py | 125 +++++++++++++------------- 1 file changed, 64 insertions(+), 61 deletions(-) diff --git a/dags/stat/community/community_rate.py b/dags/stat/community/community_rate.py index f93785e..c608b3a 100644 --- a/dags/stat/community/community_rate.py +++ b/dags/stat/community/community_rate.py @@ -14,6 +14,8 @@ from sklearn.mixture import GaussianMixture from sklearn.neighbors import BallTree from sklearn.preprocessing import MinMaxScaler from sklearn.preprocessing import StandardScaler +import multiprocessing + logger = logging.getLogger(__name__) seaweed_client = MAIN_MONGO_CLIENT.get_collection("fangjia", "seaweed") @@ -474,73 +476,74 @@ def load_data_by_city(city): data['volume_price_ratio'] = data['volume_price_ratio'].fillna(data['volume_price_ratio'].mean()) return data +import multiprocessing + +def process_city(city): + try: + logger.info(f"开始计算楼盘评级:{city}") + data = load_data_by_city(city) + data = comms_rating(data) + data["comm_id"] = data['_id'] + + db_fields = ["comm_id", "city", "region", "name", 'block', 'block_rating', "address", "sqmetre_rent", "yoy", + "rent_ro_price_ratio", "avg_price", "total_score", "community_quality_score", + "community_price_score", "location_advantage_score", "flowability_score", "living_facilities_score", + "education_score", "hospital_score", "rating", "community_quality_rating", "community_price_rating", + "location_advantage_rating", "flowability_rating", "living_facilities_rating", "education_rating", + "hospital_rating", "supermarket_num", "park_num", "catering_num", "mall_num", "kindergarten_num", + "key_primary_school_num", "ordinary_primary_school_num", "key_middle_school_num", + "ordinary_middle_school_num", "top_three_hospital_num", "non_3a_hospital_num", "build_year", "age", + "cubage_percent", "virescence_percent", "estate_charge", 'estate_type'] + + df_simple = data[db_fields].copy() + if "block" not in df_simple.columns: + df_simple['block'] = None + + for factor_score_field in ['total_score', 'community_quality_score', 'community_price_score', + 'location_advantage_score', 'flowability_score', 'living_facilities_score', + 'education_score', 'hospital_score']: + key = "" if factor_score_field == "total_score" else f"_{factor_score_field.replace('_score', '')}" + + df_simple[f'city{key}_rank'] = df_simple.groupby('city')[factor_score_field].rank( + method='dense', ascending=False).astype(int) + df_simple[f'city{key}_total_rank'] = df_simple[f'city{key}_rank'].max() + + df_simple[f'region{key}_rank'] = df_simple.groupby(['city', 'region'])[factor_score_field].rank( + method='dense', ascending=False).astype(int) + df_simple[f'region{key}_total_rank'] = df_simple[f'region{key}_rank'].max() + + df_simple[f'block{key}_rank'] = None + df_simple[f'block{key}_total_rank'] = None + if df_simple['block'].notna().any(): + subset = df_simple[df_simple['block'].notna()] + df_simple.loc[subset.index, f'block{key}_rank'] = subset.groupby( + ['city', 'region', 'block'])[factor_score_field].rank(method='dense', ascending=False).astype(int) + df_simple.loc[subset.index, f'block{key}_total_rank'] = df_simple.loc[subset.index][f'block{key}_rank'].max() + + for score_factor in ['total_score', 'community_quality_score', 'community_price_score', + 'location_advantage_score', 'flowability_score', 'living_facilities_score', + 'education_score', 'hospital_score']: + df_simple[score_factor] = df_simple[score_factor].apply(lambda x: round(x, 2)) + + df_simple['rent_ro_price_ratio'] = df_simple['rent_ro_price_ratio'].apply(lambda x: round(x, 2)) + df_simple['avg_price'] = df_simple['avg_price'].astype(int) + + updater = MongoDataFrameUpdater(community_rate_client) + updater.bulk_insert_or_update_by_id(df_simple, id_fields=['comm_id']) + logger.info(f"楼盘评级计算结束:{city}") + except Exception as e: + logging.error(f"{city}楼盘评级计算失败;{e}") + if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('--city', type=str, default='all') args = parser.parse_args() + if args.city == 'all': cities = areautils.get_all_cities() else: - cities = [city for city in args.city.split(',')] + cities = [city.strip() for city in args.city.split(',') if city.strip()] - db_fields = ["comm_id", "city", "region", "name", 'block', 'block_rating', "address", "sqmetre_rent", "yoy", - "rent_ro_price_ratio", "avg_price", "total_score", "community_quality_score", - "community_price_score", - "location_advantage_score", "flowability_score", "living_facilities_score", "education_score", - "hospital_score", "rating", "community_quality_rating", "community_price_rating", - "location_advantage_rating", "flowability_rating", "living_facilities_rating", - "education_rating", "hospital_rating", "supermarket_num", "park_num", "catering_num", "mall_num", - "kindergarten_num", - "key_primary_school_num", "ordinary_primary_school_num", "key_middle_school_num", - "ordinary_middle_school_num", "top_three_hospital_num", "non_3a_hospital_num", "build_year", "age", - "cubage_percent", "virescence_percent", "estate_charge", 'estate_type'] - for city in cities: - try: - logger.info(f"开始计算楼盘评级:{city}") - data = load_data_by_city(city) - data = comms_rating(data) - data["comm_id"] = data['_id'] - db_df = data[db_fields] - df_simple = db_df.copy() - if "block" not in df_simple.columns: - df_simple['block'] = None - for factor_score_field in ['total_score', 'community_quality_score', 'community_price_score', - 'location_advantage_score', 'flowability_score', 'living_facilities_score', - 'education_score', 'hospital_score']: - key = "_" - if factor_score_field != "total_score": - key = f"_{factor_score_field.replace('_score', '')}" - else: - key = "" - - df_simple[f'city{key}_rank'] = df_simple.groupby('city')[factor_score_field].rank(method='dense', - ascending=False).astype( - int) - df_simple[f'city{key}_total_rank'] = df_simple[f'city{key}_rank'].max() - df_simple[f'region{key}_rank'] = df_simple.groupby(['city', 'region'])[factor_score_field].rank( - method='dense', ascending=False).astype(int) - df_simple[f'region{key}_total_rank'] = df_simple[f'region{key}_rank'].max() - # 处理bis为None的情况 - 只对非空bis值计算排名 - df_simple[f'block{key}_total_rank'] = None - df_simple[f'block{key}_rank'] = None - # 对bis不为None的记录计算商圈排名 - bis_not_null = df_simple['block'].notna() - if bis_not_null.any(): - df_simple.loc[bis_not_null, f'block{key}_rank'] = \ - df_simple.loc[bis_not_null].groupby(['city', 'region', 'block'])[factor_score_field].rank( - method='dense', ascending=False).astype(int) - df_simple.loc[bis_not_null, f'block{key}_total_rank'] = df_simple.loc[bis_not_null][ - f'block{key}_rank'].max() - for score_factor in ['total_score', 'community_quality_score', 'community_price_score', - 'location_advantage_score', 'flowability_score', 'living_facilities_score', - 'education_score', 'hospital_score']: - df_simple[f'{score_factor}'] = df_simple[f'{score_factor}'].apply(lambda x: round(x, 2)) - - df_simple['rent_ro_price_ratio'] = df_simple['rent_ro_price_ratio'].apply(lambda x: round(x, 2)) - df_simple['avg_price'] = df_simple['avg_price'].astype(int) - updater = MongoDataFrameUpdater(community_rate_client) - updater.bulk_insert_or_update_by_id(df_simple, id_fields=['comm_id']) - logger.info(f"楼盘评级计算结束:{city}") - except Exception as e: - logging.error(f"{city}楼盘评级计算失败;{e}") \ No newline at end of file + with multiprocessing.Pool(processes=min(8, len(cities))) as pool: + pool.map(process_city, cities) \ No newline at end of file