From ddaccd7600899009a09825199fe37979657bffdd Mon Sep 17 00:00:00 2001 From: wangwenyan Date: Tue, 17 Jun 2025 16:34:03 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=A5=BC=E7=9B=98=E8=AF=84?= =?UTF-8?q?=E7=BA=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/stat/community/community_rate.py | 127 ++++++++++++++++++-------- 1 file changed, 89 insertions(+), 38 deletions(-) diff --git a/dags/stat/community/community_rate.py b/dags/stat/community/community_rate.py index dc40e6b..a16e349 100644 --- a/dags/stat/community/community_rate.py +++ b/dags/stat/community/community_rate.py @@ -16,7 +16,7 @@ from sklearn.preprocessing import MinMaxScaler from sklearn.preprocessing import StandardScaler logger = logging.getLogger(__name__) -seaweed_client = MAIN_MONGO_CLIENT.get_collection("fangjia","seaweed") +seaweed_client = MAIN_MONGO_CLIENT.get_collection("fangjia", "seaweed") stat_client = MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "district_stat") sqmetre_rent_client = MAIN_MONGO_CLIENT.get_collection("fangjia", "sqmetre_rent") community_rate_client = MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "community_rate") @@ -26,6 +26,7 @@ elastic_client = ElasticSearchClient(host="192.168.0.89", username='elastic', pa COMM_RATING_LEVEL_COUNT = 5 COMM_RATING_LEVEL_NAMES = {1: "S", 2: "A", 3: "B", 4: "C", 5: "D"} + # 定义数据预处理函数 def extract_years(text, validate=True): text = str(text) @@ -207,18 +208,23 @@ def load_comms(city): seaweed['_id'] = seaweed['_id'].apply(lambda x: str(x)) seaweed['lng'] = seaweed['lng'] / 10000000000 seaweed['lat'] = seaweed['lat'] / 10000000000 - seaweed = seaweed.rename(columns={"create_year": "build_year", "estate_type2": "estate_type"}) + seaweed = seaweed.rename( + columns={"create_year": "build_year", "estate_type2": "estate_type", "business_quarter": "block"}) + + year = datetime.now().year seaweed['build_year'] = seaweed['build_year'].apply(lambda x: extract_years(x)) + seaweed['age'] = seaweed['build_year'].apply(lambda x: (year - x) if pd.notna(x) else None) seaweed = seaweed.reset_index() - for field in ['build_year', "cubage_percent", "virescence_percent", "estate_charge"]: + for field in ['age', "cubage_percent", "virescence_percent", "estate_charge"]: seaweed[field] = seaweed[field].apply(lambda x: str2number(x)) na_num = len(seaweed[seaweed[field].isna()]) seaweed = fill_missing_by_around(seaweed, field) seaweed[field] = seaweed[field].fillna(seaweed[field].mean()) - seaweed['build_year'] = seaweed['build_year'].apply(lambda x: int(x)) + seaweed['age'] = seaweed['age'].apply(lambda x: int(x)) seaweed['cubage_percent'] = seaweed['cubage_percent'].apply(lambda x: round(x, 2)) seaweed['virescence_percent'] = seaweed['virescence_percent'].apply(lambda x: round(x, 2)) seaweed['estate_charge'] = seaweed['estate_charge'].apply(lambda x: round(x, 2)) + seaweed['age'] = seaweed['age'].astype(int) return seaweed @@ -256,14 +262,14 @@ def load_comm_stat(city): lastDayOfPreMonth = dateutils.add_months(firstDayOfThisMonth, days=-1) sqmetre_rent = pd.DataFrame(list( sqmetre_rent_client.find({"weekend": lastDayOfPreMonth, "cat": "district", "city": city}, - {"city": 1, "region": 1, "district_name": 1, "avg_price": 1, "_id": 0, - "price_to_rent": 1}))) + {"city": 1, "region": 1, "district_name": 1, "avg_price": 1, "_id": 0}))) sqmetre_rent = sqmetre_rent.rename(columns={"district_name": "name", "avg_price": "sqmetre_rent"}) stat = stat.merge(sqmetre_rent, on=['city', 'region', 'name'], how='left') - stat['price_to_rent'] = 1 / stat['price_to_rent'] - stat['price_to_rent'] = stat['price_to_rent'].fillna(stat['price_to_rent'].mean()) + stat['rent_ro_price_ratio'] = stat['sqmetre_rent'] * 12 / stat['avg_price'] * 100 + stat['rent_ro_price_ratio'] = stat['rent_ro_price_ratio'].fillna(stat['rent_ro_price_ratio'].mean()) return stat + # ## 加载小区配套数据 def load_comms_summary(city): @@ -290,13 +296,14 @@ def category_score(data, field, score_map): def numbic_score(data, field): data[f"{field}_score"] = normalize_to_01(data[field]) + estate_type_source_map = {"普通住宅": 0.5, "商住": 0, "别墅": 1} # ## 定义楼盘评级因子权重 factor_weights = { 'estate_type_score': 0.02, - 'build_year_score': 0.02, + 'age_score': 0.02, 'cubage_percent_score': 0.02, 'virescence_percent_score': 0.02, 'estate_charge_score': 0.02, @@ -306,7 +313,7 @@ factor_weights = { 'nearest_business_district_distance_score': 0.03, 'nearest_subway_station_distance_score': 0.07, 'nearest_bus_station_distance_score': 0.05, - 'price_to_rent_score': 0.1, + 'rent_ro_price_ratio_score': 0.1, 'volume_price_ratio_score': 0.1, 'supermarket_num_score': 0.02, 'park_num_score': 0.02, @@ -320,15 +327,16 @@ factor_weights = { 'top_three_hospital_num_score': 0.03, 'non_3a_hospital_num_score': 0.02 } + + # ## 楼盘评级打分 def comms_rating(data): - data['build_year'] = data['build_year'].max() - data['build_year'] - number_fields = ["build_year", "cubage_percent", "virescence_percent", "estate_charge", "avg_price", "sqmetre_rent", + number_fields = ["age", "cubage_percent", "virescence_percent", "estate_charge", "avg_price", "sqmetre_rent", "yoy", "supermarket_num", "park_num", "catering_num", "mall_num", "kindergarten_num", "key_primary_school_num", "ordinary_primary_school_num", "key_middle_school_num", "ordinary_middle_school_num", "top_three_hospital_num", "non_3a_hospital_num", "nearest_business_district_distance", "nearest_subway_station_distance", - "nearest_bus_station_distance", "price_to_rent", "volume_price_ratio"] + "nearest_bus_station_distance", "rent_ro_price_ratio", "volume_price_ratio"] for number_field in number_fields: threshold_95 = data[number_field].quantile(0.99) threshold_5 = data[number_field].quantile(0.01) @@ -339,16 +347,16 @@ def comms_rating(data): for category_field in ['estate_type']: category_score(data, category_field, estate_type_source_map) data['total_score'] = 0 - factors = ["estate_type", "build_year", "cubage_percent", "virescence_percent", "estate_charge", "avg_price", + factors = ["estate_type", "age", "cubage_percent", "virescence_percent", "estate_charge", "avg_price", "sqmetre_rent", "yoy", "nearest_business_district_distance", "nearest_subway_station_distance", - "nearest_bus_station_distance", "price_to_rent", "supermarket_num", "park_num", "catering_num", + "nearest_bus_station_distance", "rent_ro_price_ratio", "supermarket_num", "park_num", "catering_num", "mall_num", "kindergarten_num", "key_primary_school_num", "ordinary_primary_school_num", "key_middle_school_num", "ordinary_middle_school_num", "top_three_hospital_num", "non_3a_hospital_num"] for factor in factors: data['total_score'] = data[f"total_score"] + data[f"{factor}_score"] * factor_weights[f"{factor}_score"] data['total_score'] = normalize_to_01(data['total_score']) # 小区品质打分 - community_quality_factors = ["estate_type", "build_year", "cubage_percent", "virescence_percent", "estate_charge"] + community_quality_factors = ["estate_type", "age", "cubage_percent", "virescence_percent", "estate_charge"] data['community_quality_score'] = 0 for factor in community_quality_factors: data['community_quality_score'] = data[f"community_quality_score"] + data[f"{factor}_score"] * factor_weights[ @@ -370,7 +378,7 @@ def comms_rating(data): f"{factor}_score"] data['location_advantage_score'] = normalize_to_01(data['location_advantage_score']) # 流动性打分 - flowability_factors = ["price_to_rent", "volume_price_ratio"] + flowability_factors = ["rent_ro_price_ratio", "volume_price_ratio"] data['flowability_score'] = 0 for factor in flowability_factors: data['flowability_score'] = data[f"flowability_score"] + data[f"{factor}_score"] * factor_weights[ @@ -399,13 +407,43 @@ def comms_rating(data): data = data.dropna(subset=['total_score']) for score_factor in ['total_score', 'community_quality_score', 'community_price_score', 'location_advantage_score', 'flowability_score', 'living_facilities_score', 'education_score', 'hospital_score']: - data = gmm_train(COMM_RATING_LEVEL_COUNT, data, score_factor, f'{score_factor}_level') - data[f'{score_factor}_level'] = data[f'{score_factor}_level'].map(COMM_RATING_LEVEL_NAMES) + factor = score_factor.replace("_score", "") + if score_factor == 'total_score': + data = gmm_train(COMM_RATING_LEVEL_COUNT, data, score_factor, f'rating') + data[f'rating'] = data[f'rating'].map(COMM_RATING_LEVEL_NAMES) + else: + data = gmm_train(COMM_RATING_LEVEL_COUNT, data, score_factor, f'{factor}_rating') + data[f'{factor}_rating'] = data[f'{factor}_rating'].map(COMM_RATING_LEVEL_NAMES) + + buffer = [] + data_null = data[data['region'].isna()] + data_not_null = data[data['region'].notna()] + for item in data_not_null['region'].unique(): + group = data_not_null[data_not_null['region'] == item] + group = gmm_train(COMM_RATING_LEVEL_COUNT, group, 'total_score', f'region_rating') + group[f'region_rating'] = group[f'region_rating'].map(COMM_RATING_LEVEL_NAMES) + buffer.append(group) + buffer.append(data_null) + data = pd.concat(buffer) + buffer = [] + data_null = data[data['block'].isna()].copy() + data_not_null = data[data['block'].notna()].copy() + buffer.append(data_null) + if "block" in data.columns and (0 < len(data_not_null)): + data_not_null['region_block'] = data_not_null['region'] + "_" + data_not_null['block'] + for item in data_not_null['region_block'].unique(): + group = data_not_null[(data_not_null['region'] == str(item).split("_")[0]) & ( + data_not_null['block'] == str(item).split("_")[1])] + group = gmm_train(COMM_RATING_LEVEL_COUNT, group, 'total_score', 'block_rating') + group['block_rating'] = group['block_rating'].map(COMM_RATING_LEVEL_NAMES) + buffer.append(group) + data = pd.concat(buffer) for score_factor in ['total_score', 'community_quality_score', 'community_price_score', 'location_advantage_score', 'flowability_score', 'living_facilities_score', 'education_score', 'hospital_score']: data[f'{score_factor}'] = data[f'{score_factor}'].apply(lambda x: round(x * 100, 6)) return data + # ## 按照城市加载楼盘评级数据宽表 def load_data_by_city(city): @@ -417,7 +455,7 @@ def load_data_by_city(city): num_fields = ["supermarket_num", "park_num", "catering_num", "mall_num", "kindergarten_num", "key_primary_school_num", "ordinary_primary_school_num", "key_middle_school_num", "ordinary_middle_school_num", "top_three_hospital_num", "non_3a_hospital_num", - "half_on_year_sell_num"] + "avg_price"] for num_field in num_fields: data[num_field] = data[num_field].fillna(0) distance_fields = ["nearest_business_district_distance", "nearest_subway_station_distance", @@ -429,10 +467,11 @@ def load_data_by_city(city): data[distance_field] = data[distance_field].fillna(data[distance_field].max()) data[distance_field] = exponential_decay_weights(data[distance_field]) data[distance_field] = data[distance_field].apply(lambda x: round(x, 6)) - data['volume_price_ratio'] = data['half_on_year_sell_num'] / data['half_on_year'].replace(0, 1e-6) + data['volume_price_ratio'] = data['avg_price'] / data['half_on_year'].replace(0, 1e-6) data['volume_price_ratio'] = data['volume_price_ratio'].fillna(data['volume_price_ratio'].mean()) return data + if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('--city', type=str, default='all') @@ -442,11 +481,17 @@ if __name__ == '__main__': else: cities = [city for city in args.city.split(',')] - db_fields = ["comm_id", "city", "region", "name", "total_score", "community_quality_score", "community_price_score", + db_fields = ["comm_id", "city", "region", "name", 'block', 'block_rating', "address", "sqmetre_rent", "yoy", + "rent_ro_price_ratio", "avg_price", "total_score", "community_quality_score", + "community_price_score", "location_advantage_score", "flowability_score", "living_facilities_score", "education_score", - "hospital_score", "total_score_level", "community_quality_score_level", "community_price_score_level", - "location_advantage_score_level", "flowability_score_level", "living_facilities_score_level", - "education_score_level", "hospital_score_level"] + "hospital_score", "rating", "community_quality_rating", "community_price_rating", + "location_advantage_rating", "flowability_rating", "living_facilities_rating", + "education_rating", "hospital_rating", "supermarket_num", "park_num", "catering_num", "mall_num", + "kindergarten_num", + "key_primary_school_num", "ordinary_primary_school_num", "key_middle_school_num", + "ordinary_middle_school_num", "top_three_hospital_num", "non_3a_hospital_num", "build_year", "age", + "cubage_percent", "virescence_percent", "estate_charge", 'estate_type'] for city in cities: try: logger.info(f"开始计算楼盘评级:{city}") @@ -455,40 +500,46 @@ if __name__ == '__main__': data["comm_id"] = data['_id'] db_df = data[db_fields] df_simple = db_df.copy() - if "business_quarter" not in df_simple.columns: - df_simple['business_quarter'] = None + if "block" not in df_simple.columns: + df_simple['block'] = None for factor_score_field in ['total_score', 'community_quality_score', 'community_price_score', 'location_advantage_score', 'flowability_score', 'living_facilities_score', 'education_score', 'hospital_score']: key = "_" if factor_score_field != "total_score": key = f"_{factor_score_field.replace('_score', '')}" + else: + key = "" df_simple[f'city{key}_rank'] = df_simple.groupby('city')[factor_score_field].rank(method='dense', - ascending=False).astype( + ascending=False).astype( int) df_simple[f'city{key}_total_rank'] = df_simple[f'city{key}_rank'].max() df_simple[f'region{key}_rank'] = df_simple.groupby(['city', 'region'])[factor_score_field].rank( method='dense', ascending=False).astype(int) df_simple[f'region{key}_total_rank'] = df_simple[f'region{key}_rank'].max() # 处理bis为None的情况 - 只对非空bis值计算排名 - df_simple[f'business_quarter{key}_total_rank'] = None - df_simple[f'business_quarter{key}_rank'] = None + df_simple[f'block{key}_total_rank'] = None + df_simple[f'block{key}_rank'] = None # 对bis不为None的记录计算商圈排名 - bis_not_null = df_simple['business_quarter'].notna() + bis_not_null = df_simple['block'].notna() if bis_not_null.any(): - df_simple.loc[bis_not_null, f'business_quarter{key}_rank'] = \ - df_simple.loc[bis_not_null].groupby(['city', 'region', 'business_quarter'])[ - factor_score_field].rank(method='dense', ascending=False).astype(int) - df_simple.loc[bis_not_null, f'business_quarter{key}_total_rank'] = df_simple.loc[bis_not_null][ - f'business_quarter{key}_rank'].max() + df_simple.loc[bis_not_null, f'block{key}_rank'] = \ + df_simple.loc[bis_not_null].groupby(['city', 'region', 'block'])[factor_score_field].rank( + method='dense', ascending=False).astype(int) + df_simple.loc[bis_not_null, f'block{key}_total_rank'] = df_simple.loc[bis_not_null][ + f'block{key}_rank'].max() for score_factor in ['total_score', 'community_quality_score', 'community_price_score', 'location_advantage_score', 'flowability_score', 'living_facilities_score', 'education_score', 'hospital_score']: - df_simple[f'{score_factor}'] = df_simple[f'{score_factor}'].apply(lambda x: round(x * 100, 2)) + df_simple[f'{score_factor}'] = df_simple[f'{score_factor}'].apply(lambda x: round(x, 2)) + df_simple['rent_ro_price_ratio'] = df_simple['rent_ro_price_ratio'].apply(lambda x: round(x, 2)) + df_simple['avg_price'] = df_simple['avg_price'].astype(int) updater = MongoDataFrameUpdater(community_rate_client) updater.bulk_insert_or_update_by_id(df_simple, id_fields=['comm_id']) logger.info(f"楼盘评级计算结束:{city}") except Exception as e: - logging.error(f"{city}楼盘评级计算失败;{e}") \ No newline at end of file + print(e) + logging.error(f"{city}楼盘评级计算失败;{e}") + raise e