From 48ad7629a3339ec6d5112af5cf437e58fd68f298 Mon Sep 17 00:00:00 2001 From: wangwenyan Date: Mon, 16 Jun 2025 18:01:37 +0800 Subject: [PATCH 1/8] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=A5=BC=E7=9B=98?= =?UTF-8?q?=E8=AF=84=E7=BA=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/stat/community/community_rate.py | 37 ++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/dags/stat/community/community_rate.py b/dags/stat/community/community_rate.py index 3928d5b..746b969 100644 --- a/dags/stat/community/community_rate.py +++ b/dags/stat/community/community_rate.py @@ -201,7 +201,7 @@ def gmm_train(total_rate, data, source, target): def load_comms(city): query = {"cat": "district", "status": 0, "city": city} fields = {"city": 1, "region": 1, "name": 1, "address": 1, "lng": 1, "lat": 1, "estate_type2": 1, "create_year": 1, - "cubage_percent": 1, "virescence_percent": 1, "estate_charge": 1, "business_quarter": 1} + "cubage_percent": 1, "virescence_percent": 1, "estate_charge": 1, "business_quarter": 1, "business_quarter": 1} seaweed = pd.DataFrame(list(seaweed_client.find(query, fields))) seaweed = seaweed.dropna(subset=['lng', 'lat']) seaweed['_id'] = seaweed['_id'].apply(lambda x: str(x)) @@ -403,7 +403,7 @@ def comms_rating(data): data[f'{score_factor}_level'] = data[f'{score_factor}_level'].map(COMM_RATING_LEVEL_NAMES) for score_factor in ['total_score', 'community_quality_score', 'community_price_score', 'location_advantage_score', 'flowability_score', 'living_facilities_score', 'education_score', 'hospital_score']: - data[f'{score_factor}'] = data[f'{score_factor}'].apply(lambda x: int(x * 100)) + data[f'{score_factor}'] = data[f'{score_factor}'].apply(lambda x: round(x * 100, 6)) return data # ## 按照城市加载楼盘评级数据宽表 @@ -442,7 +442,7 @@ if __name__ == '__main__': else: cities = [city for city in args.city.split(',')] - db_fields = ["comm_id", "city", "region", "name", "total_score", "community_quality_score", "community_price_score", + db_fields = ["comm_id", "city", "region", 'business_quarter', "name", "total_score", "community_quality_score", "community_price_score", "location_advantage_score", "flowability_score", "living_facilities_score", "education_score", "hospital_score", "total_score_level", "community_quality_score_level", "community_price_score_level", "location_advantage_score_level", "flowability_score_level", "living_facilities_score_level", @@ -454,6 +454,37 @@ if __name__ == '__main__': data = comms_rating(data) data["comm_id"] = data['_id'] db_df = data[db_fields] + df_simple = db_df.copy() + for factor_score_field in ['total_score', 'community_quality_score', 'community_price_score', + 'location_advantage_score', 'flowability_score', 'living_facilities_score', + 'education_score', 'hospital_score']: + key = "_" + if factor_score_field != "total_score": + key = f"_{factor_score_field.replace('_score', '')}" + + df_simple[f'city{key}rank'] = df_simple.groupby('city')[factor_score_field].rank(method='dense', + ascending=False).astype( + int) + df_simple[f'city{key}total_rank'] = df_simple[f'city{key}rank'].max() + df_simple[f'region{key}rank'] = df_simple.groupby(['city', 'region'])[factor_score_field].rank( + method='dense', ascending=False).astype(int) + df_simple[f'region{key}total_rank'] = df_simple[f'region{key}rank'].max() + # 处理bis为None的情况 - 只对非空bis值计算排名 + df_simple[f'business_quarter{key}_total_rank'] = None + df_simple[f'business_quarter{key}_rank'] = None + # 对bis不为None的记录计算商圈排名 + bis_not_null = df_simple['business_quarter'].notna() + if bis_not_null.any(): + df_simple.loc[bis_not_null, f'business_quarter{key}_rank'] = \ + df_simple.loc[bis_not_null].groupby(['city', 'region', 'business_quarter'])[ + factor_score_field].rank(method='dense', ascending=False).astype(int) + df_simple.loc[bis_not_null, f'business_quarter{key}_total_rank'] = df_simple.loc[bis_not_null][ + f'business_quarter{key}_rank'].max() + for score_factor in ['total_score', 'community_quality_score', 'community_price_score', + 'location_advantage_score', 'flowability_score', 'living_facilities_score', + 'education_score', 'hospital_score']: + df_simple[f'{score_factor}'] = df_simple[f'{score_factor}'].apply(lambda x: round(x * 100, 2)) + updater = MongoDataFrameUpdater(community_rate_client) updater.bulk_insert_or_update_by_id(db_df, id_fields=['comm_id']) logger.info(f"楼盘评级计算结束:{city}") From 019d59ef51f95ba017a01ea0b04727cec5375101 Mon Sep 17 00:00:00 2001 From: wangwenyan Date: Mon, 16 Jun 2025 18:08:18 +0800 Subject: [PATCH 2/8] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=A5=BC=E7=9B=98?= =?UTF-8?q?=E8=AF=84=E7=BA=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/stat/community/community_rate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dags/stat/community/community_rate.py b/dags/stat/community/community_rate.py index 746b969..6b87627 100644 --- a/dags/stat/community/community_rate.py +++ b/dags/stat/community/community_rate.py @@ -201,7 +201,7 @@ def gmm_train(total_rate, data, source, target): def load_comms(city): query = {"cat": "district", "status": 0, "city": city} fields = {"city": 1, "region": 1, "name": 1, "address": 1, "lng": 1, "lat": 1, "estate_type2": 1, "create_year": 1, - "cubage_percent": 1, "virescence_percent": 1, "estate_charge": 1, "business_quarter": 1, "business_quarter": 1} + "cubage_percent": 1, "virescence_percent": 1, "estate_charge": 1, "business_quarter": 1} seaweed = pd.DataFrame(list(seaweed_client.find(query, fields))) seaweed = seaweed.dropna(subset=['lng', 'lat']) seaweed['_id'] = seaweed['_id'].apply(lambda x: str(x)) From 3a7c6b683c4771913508c9c2ac3ae53320eff723 Mon Sep 17 00:00:00 2001 From: wangwenyan Date: Mon, 16 Jun 2025 18:11:52 +0800 Subject: [PATCH 3/8] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=A5=BC=E7=9B=98?= =?UTF-8?q?=E8=AF=84=E7=BA=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/stat/community/community_rate.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dags/stat/community/community_rate.py b/dags/stat/community/community_rate.py index 6b87627..58c1e71 100644 --- a/dags/stat/community/community_rate.py +++ b/dags/stat/community/community_rate.py @@ -455,6 +455,8 @@ if __name__ == '__main__': data["comm_id"] = data['_id'] db_df = data[db_fields] df_simple = db_df.copy() + if "business_quarter" not in df_simple.columns: + df_simple['business_quarter'] = None for factor_score_field in ['total_score', 'community_quality_score', 'community_price_score', 'location_advantage_score', 'flowability_score', 'living_facilities_score', 'education_score', 'hospital_score']: @@ -486,7 +488,7 @@ if __name__ == '__main__': df_simple[f'{score_factor}'] = df_simple[f'{score_factor}'].apply(lambda x: round(x * 100, 2)) updater = MongoDataFrameUpdater(community_rate_client) - updater.bulk_insert_or_update_by_id(db_df, id_fields=['comm_id']) + updater.bulk_insert_or_update_by_id(df_simple, id_fields=['comm_id']) logger.info(f"楼盘评级计算结束:{city}") except Exception as e: logging.error(f"{city}楼盘评级计算失败;{e}") \ No newline at end of file From 22c2711485aa4e7504224ef920814ee195be09c7 Mon Sep 17 00:00:00 2001 From: wangwenyan Date: Mon, 16 Jun 2025 18:32:44 +0800 Subject: [PATCH 4/8] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=A5=BC=E7=9B=98?= =?UTF-8?q?=E8=AF=84=E7=BA=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/stat/community/community_rate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dags/stat/community/community_rate.py b/dags/stat/community/community_rate.py index 58c1e71..4ffa377 100644 --- a/dags/stat/community/community_rate.py +++ b/dags/stat/community/community_rate.py @@ -442,7 +442,7 @@ if __name__ == '__main__': else: cities = [city for city in args.city.split(',')] - db_fields = ["comm_id", "city", "region", 'business_quarter', "name", "total_score", "community_quality_score", "community_price_score", + db_fields = ["comm_id", "city", "region", "name", "total_score", "community_quality_score", "community_price_score", "location_advantage_score", "flowability_score", "living_facilities_score", "education_score", "hospital_score", "total_score_level", "community_quality_score_level", "community_price_score_level", "location_advantage_score_level", "flowability_score_level", "living_facilities_score_level", From 36b4cc59a99052c7a712ebd711907f3c1709211d Mon Sep 17 00:00:00 2001 From: wangwenyan Date: Mon, 16 Jun 2025 18:38:58 +0800 Subject: [PATCH 5/8] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=A5=BC=E7=9B=98?= =?UTF-8?q?=E8=AF=84=E7=BA=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/stat/community/community_rate.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dags/stat/community/community_rate.py b/dags/stat/community/community_rate.py index 4ffa377..dc40e6b 100644 --- a/dags/stat/community/community_rate.py +++ b/dags/stat/community/community_rate.py @@ -464,13 +464,13 @@ if __name__ == '__main__': if factor_score_field != "total_score": key = f"_{factor_score_field.replace('_score', '')}" - df_simple[f'city{key}rank'] = df_simple.groupby('city')[factor_score_field].rank(method='dense', + df_simple[f'city{key}_rank'] = df_simple.groupby('city')[factor_score_field].rank(method='dense', ascending=False).astype( int) - df_simple[f'city{key}total_rank'] = df_simple[f'city{key}rank'].max() - df_simple[f'region{key}rank'] = df_simple.groupby(['city', 'region'])[factor_score_field].rank( + df_simple[f'city{key}_total_rank'] = df_simple[f'city{key}_rank'].max() + df_simple[f'region{key}_rank'] = df_simple.groupby(['city', 'region'])[factor_score_field].rank( method='dense', ascending=False).astype(int) - df_simple[f'region{key}total_rank'] = df_simple[f'region{key}rank'].max() + df_simple[f'region{key}_total_rank'] = df_simple[f'region{key}_rank'].max() # 处理bis为None的情况 - 只对非空bis值计算排名 df_simple[f'business_quarter{key}_total_rank'] = None df_simple[f'business_quarter{key}_rank'] = None From ddaccd7600899009a09825199fe37979657bffdd Mon Sep 17 00:00:00 2001 From: wangwenyan Date: Tue, 17 Jun 2025 16:34:03 +0800 Subject: [PATCH 6/8] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=A5=BC=E7=9B=98?= =?UTF-8?q?=E8=AF=84=E7=BA=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/stat/community/community_rate.py | 127 ++++++++++++++++++-------- 1 file changed, 89 insertions(+), 38 deletions(-) diff --git a/dags/stat/community/community_rate.py b/dags/stat/community/community_rate.py index dc40e6b..a16e349 100644 --- a/dags/stat/community/community_rate.py +++ b/dags/stat/community/community_rate.py @@ -16,7 +16,7 @@ from sklearn.preprocessing import MinMaxScaler from sklearn.preprocessing import StandardScaler logger = logging.getLogger(__name__) -seaweed_client = MAIN_MONGO_CLIENT.get_collection("fangjia","seaweed") +seaweed_client = MAIN_MONGO_CLIENT.get_collection("fangjia", "seaweed") stat_client = MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "district_stat") sqmetre_rent_client = MAIN_MONGO_CLIENT.get_collection("fangjia", "sqmetre_rent") community_rate_client = MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "community_rate") @@ -26,6 +26,7 @@ elastic_client = ElasticSearchClient(host="192.168.0.89", username='elastic', pa COMM_RATING_LEVEL_COUNT = 5 COMM_RATING_LEVEL_NAMES = {1: "S", 2: "A", 3: "B", 4: "C", 5: "D"} + # 定义数据预处理函数 def extract_years(text, validate=True): text = str(text) @@ -207,18 +208,23 @@ def load_comms(city): seaweed['_id'] = seaweed['_id'].apply(lambda x: str(x)) seaweed['lng'] = seaweed['lng'] / 10000000000 seaweed['lat'] = seaweed['lat'] / 10000000000 - seaweed = seaweed.rename(columns={"create_year": "build_year", "estate_type2": "estate_type"}) + seaweed = seaweed.rename( + columns={"create_year": "build_year", "estate_type2": "estate_type", "business_quarter": "block"}) + + year = datetime.now().year seaweed['build_year'] = seaweed['build_year'].apply(lambda x: extract_years(x)) + seaweed['age'] = seaweed['build_year'].apply(lambda x: (year - x) if pd.notna(x) else None) seaweed = seaweed.reset_index() - for field in ['build_year', "cubage_percent", "virescence_percent", "estate_charge"]: + for field in ['age', "cubage_percent", "virescence_percent", "estate_charge"]: seaweed[field] = seaweed[field].apply(lambda x: str2number(x)) na_num = len(seaweed[seaweed[field].isna()]) seaweed = fill_missing_by_around(seaweed, field) seaweed[field] = seaweed[field].fillna(seaweed[field].mean()) - seaweed['build_year'] = seaweed['build_year'].apply(lambda x: int(x)) + seaweed['age'] = seaweed['age'].apply(lambda x: int(x)) seaweed['cubage_percent'] = seaweed['cubage_percent'].apply(lambda x: round(x, 2)) seaweed['virescence_percent'] = seaweed['virescence_percent'].apply(lambda x: round(x, 2)) seaweed['estate_charge'] = seaweed['estate_charge'].apply(lambda x: round(x, 2)) + seaweed['age'] = seaweed['age'].astype(int) return seaweed @@ -256,14 +262,14 @@ def load_comm_stat(city): lastDayOfPreMonth = dateutils.add_months(firstDayOfThisMonth, days=-1) sqmetre_rent = pd.DataFrame(list( sqmetre_rent_client.find({"weekend": lastDayOfPreMonth, "cat": "district", "city": city}, - {"city": 1, "region": 1, "district_name": 1, "avg_price": 1, "_id": 0, - "price_to_rent": 1}))) + {"city": 1, "region": 1, "district_name": 1, "avg_price": 1, "_id": 0}))) sqmetre_rent = sqmetre_rent.rename(columns={"district_name": "name", "avg_price": "sqmetre_rent"}) stat = stat.merge(sqmetre_rent, on=['city', 'region', 'name'], how='left') - stat['price_to_rent'] = 1 / stat['price_to_rent'] - stat['price_to_rent'] = stat['price_to_rent'].fillna(stat['price_to_rent'].mean()) + stat['rent_ro_price_ratio'] = stat['sqmetre_rent'] * 12 / stat['avg_price'] * 100 + stat['rent_ro_price_ratio'] = stat['rent_ro_price_ratio'].fillna(stat['rent_ro_price_ratio'].mean()) return stat + # ## 加载小区配套数据 def load_comms_summary(city): @@ -290,13 +296,14 @@ def category_score(data, field, score_map): def numbic_score(data, field): data[f"{field}_score"] = normalize_to_01(data[field]) + estate_type_source_map = {"普通住宅": 0.5, "商住": 0, "别墅": 1} # ## 定义楼盘评级因子权重 factor_weights = { 'estate_type_score': 0.02, - 'build_year_score': 0.02, + 'age_score': 0.02, 'cubage_percent_score': 0.02, 'virescence_percent_score': 0.02, 'estate_charge_score': 0.02, @@ -306,7 +313,7 @@ factor_weights = { 'nearest_business_district_distance_score': 0.03, 'nearest_subway_station_distance_score': 0.07, 'nearest_bus_station_distance_score': 0.05, - 'price_to_rent_score': 0.1, + 'rent_ro_price_ratio_score': 0.1, 'volume_price_ratio_score': 0.1, 'supermarket_num_score': 0.02, 'park_num_score': 0.02, @@ -320,15 +327,16 @@ factor_weights = { 'top_three_hospital_num_score': 0.03, 'non_3a_hospital_num_score': 0.02 } + + # ## 楼盘评级打分 def comms_rating(data): - data['build_year'] = data['build_year'].max() - data['build_year'] - number_fields = ["build_year", "cubage_percent", "virescence_percent", "estate_charge", "avg_price", "sqmetre_rent", + number_fields = ["age", "cubage_percent", "virescence_percent", "estate_charge", "avg_price", "sqmetre_rent", "yoy", "supermarket_num", "park_num", "catering_num", "mall_num", "kindergarten_num", "key_primary_school_num", "ordinary_primary_school_num", "key_middle_school_num", "ordinary_middle_school_num", "top_three_hospital_num", "non_3a_hospital_num", "nearest_business_district_distance", "nearest_subway_station_distance", - "nearest_bus_station_distance", "price_to_rent", "volume_price_ratio"] + "nearest_bus_station_distance", "rent_ro_price_ratio", "volume_price_ratio"] for number_field in number_fields: threshold_95 = data[number_field].quantile(0.99) threshold_5 = data[number_field].quantile(0.01) @@ -339,16 +347,16 @@ def comms_rating(data): for category_field in ['estate_type']: category_score(data, category_field, estate_type_source_map) data['total_score'] = 0 - factors = ["estate_type", "build_year", "cubage_percent", "virescence_percent", "estate_charge", "avg_price", + factors = ["estate_type", "age", "cubage_percent", "virescence_percent", "estate_charge", "avg_price", "sqmetre_rent", "yoy", "nearest_business_district_distance", "nearest_subway_station_distance", - "nearest_bus_station_distance", "price_to_rent", "supermarket_num", "park_num", "catering_num", + "nearest_bus_station_distance", "rent_ro_price_ratio", "supermarket_num", "park_num", "catering_num", "mall_num", "kindergarten_num", "key_primary_school_num", "ordinary_primary_school_num", "key_middle_school_num", "ordinary_middle_school_num", "top_three_hospital_num", "non_3a_hospital_num"] for factor in factors: data['total_score'] = data[f"total_score"] + data[f"{factor}_score"] * factor_weights[f"{factor}_score"] data['total_score'] = normalize_to_01(data['total_score']) # 小区品质打分 - community_quality_factors = ["estate_type", "build_year", "cubage_percent", "virescence_percent", "estate_charge"] + community_quality_factors = ["estate_type", "age", "cubage_percent", "virescence_percent", "estate_charge"] data['community_quality_score'] = 0 for factor in community_quality_factors: data['community_quality_score'] = data[f"community_quality_score"] + data[f"{factor}_score"] * factor_weights[ @@ -370,7 +378,7 @@ def comms_rating(data): f"{factor}_score"] data['location_advantage_score'] = normalize_to_01(data['location_advantage_score']) # 流动性打分 - flowability_factors = ["price_to_rent", "volume_price_ratio"] + flowability_factors = ["rent_ro_price_ratio", "volume_price_ratio"] data['flowability_score'] = 0 for factor in flowability_factors: data['flowability_score'] = data[f"flowability_score"] + data[f"{factor}_score"] * factor_weights[ @@ -399,13 +407,43 @@ def comms_rating(data): data = data.dropna(subset=['total_score']) for score_factor in ['total_score', 'community_quality_score', 'community_price_score', 'location_advantage_score', 'flowability_score', 'living_facilities_score', 'education_score', 'hospital_score']: - data = gmm_train(COMM_RATING_LEVEL_COUNT, data, score_factor, f'{score_factor}_level') - data[f'{score_factor}_level'] = data[f'{score_factor}_level'].map(COMM_RATING_LEVEL_NAMES) + factor = score_factor.replace("_score", "") + if score_factor == 'total_score': + data = gmm_train(COMM_RATING_LEVEL_COUNT, data, score_factor, f'rating') + data[f'rating'] = data[f'rating'].map(COMM_RATING_LEVEL_NAMES) + else: + data = gmm_train(COMM_RATING_LEVEL_COUNT, data, score_factor, f'{factor}_rating') + data[f'{factor}_rating'] = data[f'{factor}_rating'].map(COMM_RATING_LEVEL_NAMES) + + buffer = [] + data_null = data[data['region'].isna()] + data_not_null = data[data['region'].notna()] + for item in data_not_null['region'].unique(): + group = data_not_null[data_not_null['region'] == item] + group = gmm_train(COMM_RATING_LEVEL_COUNT, group, 'total_score', f'region_rating') + group[f'region_rating'] = group[f'region_rating'].map(COMM_RATING_LEVEL_NAMES) + buffer.append(group) + buffer.append(data_null) + data = pd.concat(buffer) + buffer = [] + data_null = data[data['block'].isna()].copy() + data_not_null = data[data['block'].notna()].copy() + buffer.append(data_null) + if "block" in data.columns and (0 < len(data_not_null)): + data_not_null['region_block'] = data_not_null['region'] + "_" + data_not_null['block'] + for item in data_not_null['region_block'].unique(): + group = data_not_null[(data_not_null['region'] == str(item).split("_")[0]) & ( + data_not_null['block'] == str(item).split("_")[1])] + group = gmm_train(COMM_RATING_LEVEL_COUNT, group, 'total_score', 'block_rating') + group['block_rating'] = group['block_rating'].map(COMM_RATING_LEVEL_NAMES) + buffer.append(group) + data = pd.concat(buffer) for score_factor in ['total_score', 'community_quality_score', 'community_price_score', 'location_advantage_score', 'flowability_score', 'living_facilities_score', 'education_score', 'hospital_score']: data[f'{score_factor}'] = data[f'{score_factor}'].apply(lambda x: round(x * 100, 6)) return data + # ## 按照城市加载楼盘评级数据宽表 def load_data_by_city(city): @@ -417,7 +455,7 @@ def load_data_by_city(city): num_fields = ["supermarket_num", "park_num", "catering_num", "mall_num", "kindergarten_num", "key_primary_school_num", "ordinary_primary_school_num", "key_middle_school_num", "ordinary_middle_school_num", "top_three_hospital_num", "non_3a_hospital_num", - "half_on_year_sell_num"] + "avg_price"] for num_field in num_fields: data[num_field] = data[num_field].fillna(0) distance_fields = ["nearest_business_district_distance", "nearest_subway_station_distance", @@ -429,10 +467,11 @@ def load_data_by_city(city): data[distance_field] = data[distance_field].fillna(data[distance_field].max()) data[distance_field] = exponential_decay_weights(data[distance_field]) data[distance_field] = data[distance_field].apply(lambda x: round(x, 6)) - data['volume_price_ratio'] = data['half_on_year_sell_num'] / data['half_on_year'].replace(0, 1e-6) + data['volume_price_ratio'] = data['avg_price'] / data['half_on_year'].replace(0, 1e-6) data['volume_price_ratio'] = data['volume_price_ratio'].fillna(data['volume_price_ratio'].mean()) return data + if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('--city', type=str, default='all') @@ -442,11 +481,17 @@ if __name__ == '__main__': else: cities = [city for city in args.city.split(',')] - db_fields = ["comm_id", "city", "region", "name", "total_score", "community_quality_score", "community_price_score", + db_fields = ["comm_id", "city", "region", "name", 'block', 'block_rating', "address", "sqmetre_rent", "yoy", + "rent_ro_price_ratio", "avg_price", "total_score", "community_quality_score", + "community_price_score", "location_advantage_score", "flowability_score", "living_facilities_score", "education_score", - "hospital_score", "total_score_level", "community_quality_score_level", "community_price_score_level", - "location_advantage_score_level", "flowability_score_level", "living_facilities_score_level", - "education_score_level", "hospital_score_level"] + "hospital_score", "rating", "community_quality_rating", "community_price_rating", + "location_advantage_rating", "flowability_rating", "living_facilities_rating", + "education_rating", "hospital_rating", "supermarket_num", "park_num", "catering_num", "mall_num", + "kindergarten_num", + "key_primary_school_num", "ordinary_primary_school_num", "key_middle_school_num", + "ordinary_middle_school_num", "top_three_hospital_num", "non_3a_hospital_num", "build_year", "age", + "cubage_percent", "virescence_percent", "estate_charge", 'estate_type'] for city in cities: try: logger.info(f"开始计算楼盘评级:{city}") @@ -455,40 +500,46 @@ if __name__ == '__main__': data["comm_id"] = data['_id'] db_df = data[db_fields] df_simple = db_df.copy() - if "business_quarter" not in df_simple.columns: - df_simple['business_quarter'] = None + if "block" not in df_simple.columns: + df_simple['block'] = None for factor_score_field in ['total_score', 'community_quality_score', 'community_price_score', 'location_advantage_score', 'flowability_score', 'living_facilities_score', 'education_score', 'hospital_score']: key = "_" if factor_score_field != "total_score": key = f"_{factor_score_field.replace('_score', '')}" + else: + key = "" df_simple[f'city{key}_rank'] = df_simple.groupby('city')[factor_score_field].rank(method='dense', - ascending=False).astype( + ascending=False).astype( int) df_simple[f'city{key}_total_rank'] = df_simple[f'city{key}_rank'].max() df_simple[f'region{key}_rank'] = df_simple.groupby(['city', 'region'])[factor_score_field].rank( method='dense', ascending=False).astype(int) df_simple[f'region{key}_total_rank'] = df_simple[f'region{key}_rank'].max() # 处理bis为None的情况 - 只对非空bis值计算排名 - df_simple[f'business_quarter{key}_total_rank'] = None - df_simple[f'business_quarter{key}_rank'] = None + df_simple[f'block{key}_total_rank'] = None + df_simple[f'block{key}_rank'] = None # 对bis不为None的记录计算商圈排名 - bis_not_null = df_simple['business_quarter'].notna() + bis_not_null = df_simple['block'].notna() if bis_not_null.any(): - df_simple.loc[bis_not_null, f'business_quarter{key}_rank'] = \ - df_simple.loc[bis_not_null].groupby(['city', 'region', 'business_quarter'])[ - factor_score_field].rank(method='dense', ascending=False).astype(int) - df_simple.loc[bis_not_null, f'business_quarter{key}_total_rank'] = df_simple.loc[bis_not_null][ - f'business_quarter{key}_rank'].max() + df_simple.loc[bis_not_null, f'block{key}_rank'] = \ + df_simple.loc[bis_not_null].groupby(['city', 'region', 'block'])[factor_score_field].rank( + method='dense', ascending=False).astype(int) + df_simple.loc[bis_not_null, f'block{key}_total_rank'] = df_simple.loc[bis_not_null][ + f'block{key}_rank'].max() for score_factor in ['total_score', 'community_quality_score', 'community_price_score', 'location_advantage_score', 'flowability_score', 'living_facilities_score', 'education_score', 'hospital_score']: - df_simple[f'{score_factor}'] = df_simple[f'{score_factor}'].apply(lambda x: round(x * 100, 2)) + df_simple[f'{score_factor}'] = df_simple[f'{score_factor}'].apply(lambda x: round(x, 2)) + df_simple['rent_ro_price_ratio'] = df_simple['rent_ro_price_ratio'].apply(lambda x: round(x, 2)) + df_simple['avg_price'] = df_simple['avg_price'].astype(int) updater = MongoDataFrameUpdater(community_rate_client) updater.bulk_insert_or_update_by_id(df_simple, id_fields=['comm_id']) logger.info(f"楼盘评级计算结束:{city}") except Exception as e: - logging.error(f"{city}楼盘评级计算失败;{e}") \ No newline at end of file + print(e) + logging.error(f"{city}楼盘评级计算失败;{e}") + raise e From 1f04a0a1984335cd2abba3bc930ad9f71116bb6a Mon Sep 17 00:00:00 2001 From: wangwenyan Date: Tue, 17 Jun 2025 17:07:55 +0800 Subject: [PATCH 7/8] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=A5=BC=E7=9B=98?= =?UTF-8?q?=E8=AF=84=E7=BA=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/stat/community/community_rate.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/dags/stat/community/community_rate.py b/dags/stat/community/community_rate.py index a16e349..e9089b2 100644 --- a/dags/stat/community/community_rate.py +++ b/dags/stat/community/community_rate.py @@ -425,19 +425,19 @@ def comms_rating(data): buffer.append(group) buffer.append(data_null) data = pd.concat(buffer) - buffer = [] - data_null = data[data['block'].isna()].copy() - data_not_null = data[data['block'].notna()].copy() - buffer.append(data_null) - if "block" in data.columns and (0 < len(data_not_null)): - data_not_null['region_block'] = data_not_null['region'] + "_" + data_not_null['block'] - for item in data_not_null['region_block'].unique(): - group = data_not_null[(data_not_null['region'] == str(item).split("_")[0]) & ( - data_not_null['block'] == str(item).split("_")[1])] - group = gmm_train(COMM_RATING_LEVEL_COUNT, group, 'total_score', 'block_rating') - group['block_rating'] = group['block_rating'].map(COMM_RATING_LEVEL_NAMES) - buffer.append(group) - data = pd.concat(buffer) + if "block" in data.columns: + buffer = [] + data_null = data[data['block'].isna()].copy() + data_not_null = data[data['block'].notna()].copy() + buffer.append(data_null) + if len(data_not_null) > 0: + data_not_null['region_block'] = data_not_null['region'] + "_" + data_not_null['block'] + for item in data_not_null['region_block'].unique(): + group = data_not_null[(data_not_null['region'] == str(item).split("_")[0]) & (data_not_null['block'] == str(item).split("_")[1])] + group = gmm_train(COMM_RATING_LEVEL_COUNT, group, 'total_score', 'block_rating') + group['block_rating'] = group['block_rating'].map(COMM_RATING_LEVEL_NAMES) + buffer.append(group) + data = pd.concat(buffer) for score_factor in ['total_score', 'community_quality_score', 'community_price_score', 'location_advantage_score', 'flowability_score', 'living_facilities_score', 'education_score', 'hospital_score']: data[f'{score_factor}'] = data[f'{score_factor}'].apply(lambda x: round(x * 100, 6)) From 235fd10e3f0b4c6ca542edd8b02b11f9f668c355 Mon Sep 17 00:00:00 2001 From: wangwenyan Date: Tue, 17 Jun 2025 17:15:51 +0800 Subject: [PATCH 8/8] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=A5=BC=E7=9B=98?= =?UTF-8?q?=E8=AF=84=E7=BA=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/stat/community/community_rate.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dags/stat/community/community_rate.py b/dags/stat/community/community_rate.py index e9089b2..77a238d 100644 --- a/dags/stat/community/community_rate.py +++ b/dags/stat/community/community_rate.py @@ -438,6 +438,9 @@ def comms_rating(data): group['block_rating'] = group['block_rating'].map(COMM_RATING_LEVEL_NAMES) buffer.append(group) data = pd.concat(buffer) + else: + data['block_rating'] = None + data['block'] = None for score_factor in ['total_score', 'community_quality_score', 'community_price_score', 'location_advantage_score', 'flowability_score', 'living_facilities_score', 'education_score', 'hospital_score']: data[f'{score_factor}'] = data[f'{score_factor}'].apply(lambda x: round(x * 100, 6))