优化楼盘评级

This commit is contained in:
2025-06-17 16:34:03 +08:00
parent 36b4cc59a9
commit ddaccd7600
+89 -38
View File
@@ -16,7 +16,7 @@ from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import StandardScaler
logger = logging.getLogger(__name__)
seaweed_client = MAIN_MONGO_CLIENT.get_collection("fangjia","seaweed")
seaweed_client = MAIN_MONGO_CLIENT.get_collection("fangjia", "seaweed")
stat_client = MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "district_stat")
sqmetre_rent_client = MAIN_MONGO_CLIENT.get_collection("fangjia", "sqmetre_rent")
community_rate_client = MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "community_rate")
@@ -26,6 +26,7 @@ elastic_client = ElasticSearchClient(host="192.168.0.89", username='elastic', pa
COMM_RATING_LEVEL_COUNT = 5
COMM_RATING_LEVEL_NAMES = {1: "S", 2: "A", 3: "B", 4: "C", 5: "D"}
# 定义数据预处理函数
def extract_years(text, validate=True):
text = str(text)
@@ -207,18 +208,23 @@ def load_comms(city):
seaweed['_id'] = seaweed['_id'].apply(lambda x: str(x))
seaweed['lng'] = seaweed['lng'] / 10000000000
seaweed['lat'] = seaweed['lat'] / 10000000000
seaweed = seaweed.rename(columns={"create_year": "build_year", "estate_type2": "estate_type"})
seaweed = seaweed.rename(
columns={"create_year": "build_year", "estate_type2": "estate_type", "business_quarter": "block"})
year = datetime.now().year
seaweed['build_year'] = seaweed['build_year'].apply(lambda x: extract_years(x))
seaweed['age'] = seaweed['build_year'].apply(lambda x: (year - x) if pd.notna(x) else None)
seaweed = seaweed.reset_index()
for field in ['build_year', "cubage_percent", "virescence_percent", "estate_charge"]:
for field in ['age', "cubage_percent", "virescence_percent", "estate_charge"]:
seaweed[field] = seaweed[field].apply(lambda x: str2number(x))
na_num = len(seaweed[seaweed[field].isna()])
seaweed = fill_missing_by_around(seaweed, field)
seaweed[field] = seaweed[field].fillna(seaweed[field].mean())
seaweed['build_year'] = seaweed['build_year'].apply(lambda x: int(x))
seaweed['age'] = seaweed['age'].apply(lambda x: int(x))
seaweed['cubage_percent'] = seaweed['cubage_percent'].apply(lambda x: round(x, 2))
seaweed['virescence_percent'] = seaweed['virescence_percent'].apply(lambda x: round(x, 2))
seaweed['estate_charge'] = seaweed['estate_charge'].apply(lambda x: round(x, 2))
seaweed['age'] = seaweed['age'].astype(int)
return seaweed
@@ -256,14 +262,14 @@ def load_comm_stat(city):
lastDayOfPreMonth = dateutils.add_months(firstDayOfThisMonth, days=-1)
sqmetre_rent = pd.DataFrame(list(
sqmetre_rent_client.find({"weekend": lastDayOfPreMonth, "cat": "district", "city": city},
{"city": 1, "region": 1, "district_name": 1, "avg_price": 1, "_id": 0,
"price_to_rent": 1})))
{"city": 1, "region": 1, "district_name": 1, "avg_price": 1, "_id": 0})))
sqmetre_rent = sqmetre_rent.rename(columns={"district_name": "name", "avg_price": "sqmetre_rent"})
stat = stat.merge(sqmetre_rent, on=['city', 'region', 'name'], how='left')
stat['price_to_rent'] = 1 / stat['price_to_rent']
stat['price_to_rent'] = stat['price_to_rent'].fillna(stat['price_to_rent'].mean())
stat['rent_ro_price_ratio'] = stat['sqmetre_rent'] * 12 / stat['avg_price'] * 100
stat['rent_ro_price_ratio'] = stat['rent_ro_price_ratio'].fillna(stat['rent_ro_price_ratio'].mean())
return stat
# ## 加载小区配套数据
def load_comms_summary(city):
@@ -290,13 +296,14 @@ def category_score(data, field, score_map):
def numbic_score(data, field):
data[f"{field}_score"] = normalize_to_01(data[field])
estate_type_source_map = {"普通住宅": 0.5, "商住": 0, "别墅": 1}
# ## 定义楼盘评级因子权重
factor_weights = {
'estate_type_score': 0.02,
'build_year_score': 0.02,
'age_score': 0.02,
'cubage_percent_score': 0.02,
'virescence_percent_score': 0.02,
'estate_charge_score': 0.02,
@@ -306,7 +313,7 @@ factor_weights = {
'nearest_business_district_distance_score': 0.03,
'nearest_subway_station_distance_score': 0.07,
'nearest_bus_station_distance_score': 0.05,
'price_to_rent_score': 0.1,
'rent_ro_price_ratio_score': 0.1,
'volume_price_ratio_score': 0.1,
'supermarket_num_score': 0.02,
'park_num_score': 0.02,
@@ -320,15 +327,16 @@ factor_weights = {
'top_three_hospital_num_score': 0.03,
'non_3a_hospital_num_score': 0.02
}
# ## 楼盘评级打分
def comms_rating(data):
data['build_year'] = data['build_year'].max() - data['build_year']
number_fields = ["build_year", "cubage_percent", "virescence_percent", "estate_charge", "avg_price", "sqmetre_rent",
number_fields = ["age", "cubage_percent", "virescence_percent", "estate_charge", "avg_price", "sqmetre_rent",
"yoy", "supermarket_num", "park_num", "catering_num", "mall_num", "kindergarten_num",
"key_primary_school_num", "ordinary_primary_school_num", "key_middle_school_num",
"ordinary_middle_school_num", "top_three_hospital_num", "non_3a_hospital_num",
"nearest_business_district_distance", "nearest_subway_station_distance",
"nearest_bus_station_distance", "price_to_rent", "volume_price_ratio"]
"nearest_bus_station_distance", "rent_ro_price_ratio", "volume_price_ratio"]
for number_field in number_fields:
threshold_95 = data[number_field].quantile(0.99)
threshold_5 = data[number_field].quantile(0.01)
@@ -339,16 +347,16 @@ def comms_rating(data):
for category_field in ['estate_type']:
category_score(data, category_field, estate_type_source_map)
data['total_score'] = 0
factors = ["estate_type", "build_year", "cubage_percent", "virescence_percent", "estate_charge", "avg_price",
factors = ["estate_type", "age", "cubage_percent", "virescence_percent", "estate_charge", "avg_price",
"sqmetre_rent", "yoy", "nearest_business_district_distance", "nearest_subway_station_distance",
"nearest_bus_station_distance", "price_to_rent", "supermarket_num", "park_num", "catering_num",
"nearest_bus_station_distance", "rent_ro_price_ratio", "supermarket_num", "park_num", "catering_num",
"mall_num", "kindergarten_num", "key_primary_school_num", "ordinary_primary_school_num",
"key_middle_school_num", "ordinary_middle_school_num", "top_three_hospital_num", "non_3a_hospital_num"]
for factor in factors:
data['total_score'] = data[f"total_score"] + data[f"{factor}_score"] * factor_weights[f"{factor}_score"]
data['total_score'] = normalize_to_01(data['total_score'])
# 小区品质打分
community_quality_factors = ["estate_type", "build_year", "cubage_percent", "virescence_percent", "estate_charge"]
community_quality_factors = ["estate_type", "age", "cubage_percent", "virescence_percent", "estate_charge"]
data['community_quality_score'] = 0
for factor in community_quality_factors:
data['community_quality_score'] = data[f"community_quality_score"] + data[f"{factor}_score"] * factor_weights[
@@ -370,7 +378,7 @@ def comms_rating(data):
f"{factor}_score"]
data['location_advantage_score'] = normalize_to_01(data['location_advantage_score'])
# 流动性打分
flowability_factors = ["price_to_rent", "volume_price_ratio"]
flowability_factors = ["rent_ro_price_ratio", "volume_price_ratio"]
data['flowability_score'] = 0
for factor in flowability_factors:
data['flowability_score'] = data[f"flowability_score"] + data[f"{factor}_score"] * factor_weights[
@@ -399,13 +407,43 @@ def comms_rating(data):
data = data.dropna(subset=['total_score'])
for score_factor in ['total_score', 'community_quality_score', 'community_price_score', 'location_advantage_score',
'flowability_score', 'living_facilities_score', 'education_score', 'hospital_score']:
data = gmm_train(COMM_RATING_LEVEL_COUNT, data, score_factor, f'{score_factor}_level')
data[f'{score_factor}_level'] = data[f'{score_factor}_level'].map(COMM_RATING_LEVEL_NAMES)
factor = score_factor.replace("_score", "")
if score_factor == 'total_score':
data = gmm_train(COMM_RATING_LEVEL_COUNT, data, score_factor, f'rating')
data[f'rating'] = data[f'rating'].map(COMM_RATING_LEVEL_NAMES)
else:
data = gmm_train(COMM_RATING_LEVEL_COUNT, data, score_factor, f'{factor}_rating')
data[f'{factor}_rating'] = data[f'{factor}_rating'].map(COMM_RATING_LEVEL_NAMES)
buffer = []
data_null = data[data['region'].isna()]
data_not_null = data[data['region'].notna()]
for item in data_not_null['region'].unique():
group = data_not_null[data_not_null['region'] == item]
group = gmm_train(COMM_RATING_LEVEL_COUNT, group, 'total_score', f'region_rating')
group[f'region_rating'] = group[f'region_rating'].map(COMM_RATING_LEVEL_NAMES)
buffer.append(group)
buffer.append(data_null)
data = pd.concat(buffer)
buffer = []
data_null = data[data['block'].isna()].copy()
data_not_null = data[data['block'].notna()].copy()
buffer.append(data_null)
if "block" in data.columns and (0 < len(data_not_null)):
data_not_null['region_block'] = data_not_null['region'] + "_" + data_not_null['block']
for item in data_not_null['region_block'].unique():
group = data_not_null[(data_not_null['region'] == str(item).split("_")[0]) & (
data_not_null['block'] == str(item).split("_")[1])]
group = gmm_train(COMM_RATING_LEVEL_COUNT, group, 'total_score', 'block_rating')
group['block_rating'] = group['block_rating'].map(COMM_RATING_LEVEL_NAMES)
buffer.append(group)
data = pd.concat(buffer)
for score_factor in ['total_score', 'community_quality_score', 'community_price_score', 'location_advantage_score',
'flowability_score', 'living_facilities_score', 'education_score', 'hospital_score']:
data[f'{score_factor}'] = data[f'{score_factor}'].apply(lambda x: round(x * 100, 6))
return data
# ## 按照城市加载楼盘评级数据宽表
def load_data_by_city(city):
@@ -417,7 +455,7 @@ def load_data_by_city(city):
num_fields = ["supermarket_num", "park_num", "catering_num", "mall_num", "kindergarten_num",
"key_primary_school_num", "ordinary_primary_school_num", "key_middle_school_num",
"ordinary_middle_school_num", "top_three_hospital_num", "non_3a_hospital_num",
"half_on_year_sell_num"]
"avg_price"]
for num_field in num_fields:
data[num_field] = data[num_field].fillna(0)
distance_fields = ["nearest_business_district_distance", "nearest_subway_station_distance",
@@ -429,10 +467,11 @@ def load_data_by_city(city):
data[distance_field] = data[distance_field].fillna(data[distance_field].max())
data[distance_field] = exponential_decay_weights(data[distance_field])
data[distance_field] = data[distance_field].apply(lambda x: round(x, 6))
data['volume_price_ratio'] = data['half_on_year_sell_num'] / data['half_on_year'].replace(0, 1e-6)
data['volume_price_ratio'] = data['avg_price'] / data['half_on_year'].replace(0, 1e-6)
data['volume_price_ratio'] = data['volume_price_ratio'].fillna(data['volume_price_ratio'].mean())
return data
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--city', type=str, default='all')
@@ -442,11 +481,17 @@ if __name__ == '__main__':
else:
cities = [city for city in args.city.split(',')]
db_fields = ["comm_id", "city", "region", "name", "total_score", "community_quality_score", "community_price_score",
db_fields = ["comm_id", "city", "region", "name", 'block', 'block_rating', "address", "sqmetre_rent", "yoy",
"rent_ro_price_ratio", "avg_price", "total_score", "community_quality_score",
"community_price_score",
"location_advantage_score", "flowability_score", "living_facilities_score", "education_score",
"hospital_score", "total_score_level", "community_quality_score_level", "community_price_score_level",
"location_advantage_score_level", "flowability_score_level", "living_facilities_score_level",
"education_score_level", "hospital_score_level"]
"hospital_score", "rating", "community_quality_rating", "community_price_rating",
"location_advantage_rating", "flowability_rating", "living_facilities_rating",
"education_rating", "hospital_rating", "supermarket_num", "park_num", "catering_num", "mall_num",
"kindergarten_num",
"key_primary_school_num", "ordinary_primary_school_num", "key_middle_school_num",
"ordinary_middle_school_num", "top_three_hospital_num", "non_3a_hospital_num", "build_year", "age",
"cubage_percent", "virescence_percent", "estate_charge", 'estate_type']
for city in cities:
try:
logger.info(f"开始计算楼盘评级:{city}")
@@ -455,40 +500,46 @@ if __name__ == '__main__':
data["comm_id"] = data['_id']
db_df = data[db_fields]
df_simple = db_df.copy()
if "business_quarter" not in df_simple.columns:
df_simple['business_quarter'] = None
if "block" not in df_simple.columns:
df_simple['block'] = None
for factor_score_field in ['total_score', 'community_quality_score', 'community_price_score',
'location_advantage_score', 'flowability_score', 'living_facilities_score',
'education_score', 'hospital_score']:
key = "_"
if factor_score_field != "total_score":
key = f"_{factor_score_field.replace('_score', '')}"
else:
key = ""
df_simple[f'city{key}_rank'] = df_simple.groupby('city')[factor_score_field].rank(method='dense',
ascending=False).astype(
ascending=False).astype(
int)
df_simple[f'city{key}_total_rank'] = df_simple[f'city{key}_rank'].max()
df_simple[f'region{key}_rank'] = df_simple.groupby(['city', 'region'])[factor_score_field].rank(
method='dense', ascending=False).astype(int)
df_simple[f'region{key}_total_rank'] = df_simple[f'region{key}_rank'].max()
# 处理bis为None的情况 - 只对非空bis值计算排名
df_simple[f'business_quarter{key}_total_rank'] = None
df_simple[f'business_quarter{key}_rank'] = None
df_simple[f'block{key}_total_rank'] = None
df_simple[f'block{key}_rank'] = None
# 对bis不为None的记录计算商圈排名
bis_not_null = df_simple['business_quarter'].notna()
bis_not_null = df_simple['block'].notna()
if bis_not_null.any():
df_simple.loc[bis_not_null, f'business_quarter{key}_rank'] = \
df_simple.loc[bis_not_null].groupby(['city', 'region', 'business_quarter'])[
factor_score_field].rank(method='dense', ascending=False).astype(int)
df_simple.loc[bis_not_null, f'business_quarter{key}_total_rank'] = df_simple.loc[bis_not_null][
f'business_quarter{key}_rank'].max()
df_simple.loc[bis_not_null, f'block{key}_rank'] = \
df_simple.loc[bis_not_null].groupby(['city', 'region', 'block'])[factor_score_field].rank(
method='dense', ascending=False).astype(int)
df_simple.loc[bis_not_null, f'block{key}_total_rank'] = df_simple.loc[bis_not_null][
f'block{key}_rank'].max()
for score_factor in ['total_score', 'community_quality_score', 'community_price_score',
'location_advantage_score', 'flowability_score', 'living_facilities_score',
'education_score', 'hospital_score']:
df_simple[f'{score_factor}'] = df_simple[f'{score_factor}'].apply(lambda x: round(x * 100, 2))
df_simple[f'{score_factor}'] = df_simple[f'{score_factor}'].apply(lambda x: round(x, 2))
df_simple['rent_ro_price_ratio'] = df_simple['rent_ro_price_ratio'].apply(lambda x: round(x, 2))
df_simple['avg_price'] = df_simple['avg_price'].astype(int)
updater = MongoDataFrameUpdater(community_rate_client)
updater.bulk_insert_or_update_by_id(df_simple, id_fields=['comm_id'])
logger.info(f"楼盘评级计算结束:{city}")
except Exception as e:
logging.error(f"{city}楼盘评级计算失败;{e}")
print(e)
logging.error(f"{city}楼盘评级计算失败;{e}")
raise e