优化楼盘评级
This commit is contained in:
@@ -16,7 +16,7 @@ from sklearn.preprocessing import MinMaxScaler
|
||||
from sklearn.preprocessing import StandardScaler
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
seaweed_client = MAIN_MONGO_CLIENT.get_collection("fangjia","seaweed")
|
||||
seaweed_client = MAIN_MONGO_CLIENT.get_collection("fangjia", "seaweed")
|
||||
stat_client = MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "district_stat")
|
||||
sqmetre_rent_client = MAIN_MONGO_CLIENT.get_collection("fangjia", "sqmetre_rent")
|
||||
community_rate_client = MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "community_rate")
|
||||
@@ -26,6 +26,7 @@ elastic_client = ElasticSearchClient(host="192.168.0.89", username='elastic', pa
|
||||
COMM_RATING_LEVEL_COUNT = 5
|
||||
COMM_RATING_LEVEL_NAMES = {1: "S", 2: "A", 3: "B", 4: "C", 5: "D"}
|
||||
|
||||
|
||||
# 定义数据预处理函数
|
||||
def extract_years(text, validate=True):
|
||||
text = str(text)
|
||||
@@ -207,18 +208,23 @@ def load_comms(city):
|
||||
seaweed['_id'] = seaweed['_id'].apply(lambda x: str(x))
|
||||
seaweed['lng'] = seaweed['lng'] / 10000000000
|
||||
seaweed['lat'] = seaweed['lat'] / 10000000000
|
||||
seaweed = seaweed.rename(columns={"create_year": "build_year", "estate_type2": "estate_type"})
|
||||
seaweed = seaweed.rename(
|
||||
columns={"create_year": "build_year", "estate_type2": "estate_type", "business_quarter": "block"})
|
||||
|
||||
year = datetime.now().year
|
||||
seaweed['build_year'] = seaweed['build_year'].apply(lambda x: extract_years(x))
|
||||
seaweed['age'] = seaweed['build_year'].apply(lambda x: (year - x) if pd.notna(x) else None)
|
||||
seaweed = seaweed.reset_index()
|
||||
for field in ['build_year', "cubage_percent", "virescence_percent", "estate_charge"]:
|
||||
for field in ['age', "cubage_percent", "virescence_percent", "estate_charge"]:
|
||||
seaweed[field] = seaweed[field].apply(lambda x: str2number(x))
|
||||
na_num = len(seaweed[seaweed[field].isna()])
|
||||
seaweed = fill_missing_by_around(seaweed, field)
|
||||
seaweed[field] = seaweed[field].fillna(seaweed[field].mean())
|
||||
seaweed['build_year'] = seaweed['build_year'].apply(lambda x: int(x))
|
||||
seaweed['age'] = seaweed['age'].apply(lambda x: int(x))
|
||||
seaweed['cubage_percent'] = seaweed['cubage_percent'].apply(lambda x: round(x, 2))
|
||||
seaweed['virescence_percent'] = seaweed['virescence_percent'].apply(lambda x: round(x, 2))
|
||||
seaweed['estate_charge'] = seaweed['estate_charge'].apply(lambda x: round(x, 2))
|
||||
seaweed['age'] = seaweed['age'].astype(int)
|
||||
return seaweed
|
||||
|
||||
|
||||
@@ -256,14 +262,14 @@ def load_comm_stat(city):
|
||||
lastDayOfPreMonth = dateutils.add_months(firstDayOfThisMonth, days=-1)
|
||||
sqmetre_rent = pd.DataFrame(list(
|
||||
sqmetre_rent_client.find({"weekend": lastDayOfPreMonth, "cat": "district", "city": city},
|
||||
{"city": 1, "region": 1, "district_name": 1, "avg_price": 1, "_id": 0,
|
||||
"price_to_rent": 1})))
|
||||
{"city": 1, "region": 1, "district_name": 1, "avg_price": 1, "_id": 0})))
|
||||
sqmetre_rent = sqmetre_rent.rename(columns={"district_name": "name", "avg_price": "sqmetre_rent"})
|
||||
stat = stat.merge(sqmetre_rent, on=['city', 'region', 'name'], how='left')
|
||||
stat['price_to_rent'] = 1 / stat['price_to_rent']
|
||||
stat['price_to_rent'] = stat['price_to_rent'].fillna(stat['price_to_rent'].mean())
|
||||
stat['rent_ro_price_ratio'] = stat['sqmetre_rent'] * 12 / stat['avg_price'] * 100
|
||||
stat['rent_ro_price_ratio'] = stat['rent_ro_price_ratio'].fillna(stat['rent_ro_price_ratio'].mean())
|
||||
return stat
|
||||
|
||||
|
||||
# ## 加载小区配套数据
|
||||
|
||||
def load_comms_summary(city):
|
||||
@@ -290,13 +296,14 @@ def category_score(data, field, score_map):
|
||||
def numbic_score(data, field):
|
||||
data[f"{field}_score"] = normalize_to_01(data[field])
|
||||
|
||||
|
||||
estate_type_source_map = {"普通住宅": 0.5, "商住": 0, "别墅": 1}
|
||||
|
||||
# ## 定义楼盘评级因子权重
|
||||
|
||||
factor_weights = {
|
||||
'estate_type_score': 0.02,
|
||||
'build_year_score': 0.02,
|
||||
'age_score': 0.02,
|
||||
'cubage_percent_score': 0.02,
|
||||
'virescence_percent_score': 0.02,
|
||||
'estate_charge_score': 0.02,
|
||||
@@ -306,7 +313,7 @@ factor_weights = {
|
||||
'nearest_business_district_distance_score': 0.03,
|
||||
'nearest_subway_station_distance_score': 0.07,
|
||||
'nearest_bus_station_distance_score': 0.05,
|
||||
'price_to_rent_score': 0.1,
|
||||
'rent_ro_price_ratio_score': 0.1,
|
||||
'volume_price_ratio_score': 0.1,
|
||||
'supermarket_num_score': 0.02,
|
||||
'park_num_score': 0.02,
|
||||
@@ -320,15 +327,16 @@ factor_weights = {
|
||||
'top_three_hospital_num_score': 0.03,
|
||||
'non_3a_hospital_num_score': 0.02
|
||||
}
|
||||
|
||||
|
||||
# ## 楼盘评级打分
|
||||
def comms_rating(data):
|
||||
data['build_year'] = data['build_year'].max() - data['build_year']
|
||||
number_fields = ["build_year", "cubage_percent", "virescence_percent", "estate_charge", "avg_price", "sqmetre_rent",
|
||||
number_fields = ["age", "cubage_percent", "virescence_percent", "estate_charge", "avg_price", "sqmetre_rent",
|
||||
"yoy", "supermarket_num", "park_num", "catering_num", "mall_num", "kindergarten_num",
|
||||
"key_primary_school_num", "ordinary_primary_school_num", "key_middle_school_num",
|
||||
"ordinary_middle_school_num", "top_three_hospital_num", "non_3a_hospital_num",
|
||||
"nearest_business_district_distance", "nearest_subway_station_distance",
|
||||
"nearest_bus_station_distance", "price_to_rent", "volume_price_ratio"]
|
||||
"nearest_bus_station_distance", "rent_ro_price_ratio", "volume_price_ratio"]
|
||||
for number_field in number_fields:
|
||||
threshold_95 = data[number_field].quantile(0.99)
|
||||
threshold_5 = data[number_field].quantile(0.01)
|
||||
@@ -339,16 +347,16 @@ def comms_rating(data):
|
||||
for category_field in ['estate_type']:
|
||||
category_score(data, category_field, estate_type_source_map)
|
||||
data['total_score'] = 0
|
||||
factors = ["estate_type", "build_year", "cubage_percent", "virescence_percent", "estate_charge", "avg_price",
|
||||
factors = ["estate_type", "age", "cubage_percent", "virescence_percent", "estate_charge", "avg_price",
|
||||
"sqmetre_rent", "yoy", "nearest_business_district_distance", "nearest_subway_station_distance",
|
||||
"nearest_bus_station_distance", "price_to_rent", "supermarket_num", "park_num", "catering_num",
|
||||
"nearest_bus_station_distance", "rent_ro_price_ratio", "supermarket_num", "park_num", "catering_num",
|
||||
"mall_num", "kindergarten_num", "key_primary_school_num", "ordinary_primary_school_num",
|
||||
"key_middle_school_num", "ordinary_middle_school_num", "top_three_hospital_num", "non_3a_hospital_num"]
|
||||
for factor in factors:
|
||||
data['total_score'] = data[f"total_score"] + data[f"{factor}_score"] * factor_weights[f"{factor}_score"]
|
||||
data['total_score'] = normalize_to_01(data['total_score'])
|
||||
# 小区品质打分
|
||||
community_quality_factors = ["estate_type", "build_year", "cubage_percent", "virescence_percent", "estate_charge"]
|
||||
community_quality_factors = ["estate_type", "age", "cubage_percent", "virescence_percent", "estate_charge"]
|
||||
data['community_quality_score'] = 0
|
||||
for factor in community_quality_factors:
|
||||
data['community_quality_score'] = data[f"community_quality_score"] + data[f"{factor}_score"] * factor_weights[
|
||||
@@ -370,7 +378,7 @@ def comms_rating(data):
|
||||
f"{factor}_score"]
|
||||
data['location_advantage_score'] = normalize_to_01(data['location_advantage_score'])
|
||||
# 流动性打分
|
||||
flowability_factors = ["price_to_rent", "volume_price_ratio"]
|
||||
flowability_factors = ["rent_ro_price_ratio", "volume_price_ratio"]
|
||||
data['flowability_score'] = 0
|
||||
for factor in flowability_factors:
|
||||
data['flowability_score'] = data[f"flowability_score"] + data[f"{factor}_score"] * factor_weights[
|
||||
@@ -399,13 +407,43 @@ def comms_rating(data):
|
||||
data = data.dropna(subset=['total_score'])
|
||||
for score_factor in ['total_score', 'community_quality_score', 'community_price_score', 'location_advantage_score',
|
||||
'flowability_score', 'living_facilities_score', 'education_score', 'hospital_score']:
|
||||
data = gmm_train(COMM_RATING_LEVEL_COUNT, data, score_factor, f'{score_factor}_level')
|
||||
data[f'{score_factor}_level'] = data[f'{score_factor}_level'].map(COMM_RATING_LEVEL_NAMES)
|
||||
factor = score_factor.replace("_score", "")
|
||||
if score_factor == 'total_score':
|
||||
data = gmm_train(COMM_RATING_LEVEL_COUNT, data, score_factor, f'rating')
|
||||
data[f'rating'] = data[f'rating'].map(COMM_RATING_LEVEL_NAMES)
|
||||
else:
|
||||
data = gmm_train(COMM_RATING_LEVEL_COUNT, data, score_factor, f'{factor}_rating')
|
||||
data[f'{factor}_rating'] = data[f'{factor}_rating'].map(COMM_RATING_LEVEL_NAMES)
|
||||
|
||||
buffer = []
|
||||
data_null = data[data['region'].isna()]
|
||||
data_not_null = data[data['region'].notna()]
|
||||
for item in data_not_null['region'].unique():
|
||||
group = data_not_null[data_not_null['region'] == item]
|
||||
group = gmm_train(COMM_RATING_LEVEL_COUNT, group, 'total_score', f'region_rating')
|
||||
group[f'region_rating'] = group[f'region_rating'].map(COMM_RATING_LEVEL_NAMES)
|
||||
buffer.append(group)
|
||||
buffer.append(data_null)
|
||||
data = pd.concat(buffer)
|
||||
buffer = []
|
||||
data_null = data[data['block'].isna()].copy()
|
||||
data_not_null = data[data['block'].notna()].copy()
|
||||
buffer.append(data_null)
|
||||
if "block" in data.columns and (0 < len(data_not_null)):
|
||||
data_not_null['region_block'] = data_not_null['region'] + "_" + data_not_null['block']
|
||||
for item in data_not_null['region_block'].unique():
|
||||
group = data_not_null[(data_not_null['region'] == str(item).split("_")[0]) & (
|
||||
data_not_null['block'] == str(item).split("_")[1])]
|
||||
group = gmm_train(COMM_RATING_LEVEL_COUNT, group, 'total_score', 'block_rating')
|
||||
group['block_rating'] = group['block_rating'].map(COMM_RATING_LEVEL_NAMES)
|
||||
buffer.append(group)
|
||||
data = pd.concat(buffer)
|
||||
for score_factor in ['total_score', 'community_quality_score', 'community_price_score', 'location_advantage_score',
|
||||
'flowability_score', 'living_facilities_score', 'education_score', 'hospital_score']:
|
||||
data[f'{score_factor}'] = data[f'{score_factor}'].apply(lambda x: round(x * 100, 6))
|
||||
return data
|
||||
|
||||
|
||||
# ## 按照城市加载楼盘评级数据宽表
|
||||
|
||||
def load_data_by_city(city):
|
||||
@@ -417,7 +455,7 @@ def load_data_by_city(city):
|
||||
num_fields = ["supermarket_num", "park_num", "catering_num", "mall_num", "kindergarten_num",
|
||||
"key_primary_school_num", "ordinary_primary_school_num", "key_middle_school_num",
|
||||
"ordinary_middle_school_num", "top_three_hospital_num", "non_3a_hospital_num",
|
||||
"half_on_year_sell_num"]
|
||||
"avg_price"]
|
||||
for num_field in num_fields:
|
||||
data[num_field] = data[num_field].fillna(0)
|
||||
distance_fields = ["nearest_business_district_distance", "nearest_subway_station_distance",
|
||||
@@ -429,10 +467,11 @@ def load_data_by_city(city):
|
||||
data[distance_field] = data[distance_field].fillna(data[distance_field].max())
|
||||
data[distance_field] = exponential_decay_weights(data[distance_field])
|
||||
data[distance_field] = data[distance_field].apply(lambda x: round(x, 6))
|
||||
data['volume_price_ratio'] = data['half_on_year_sell_num'] / data['half_on_year'].replace(0, 1e-6)
|
||||
data['volume_price_ratio'] = data['avg_price'] / data['half_on_year'].replace(0, 1e-6)
|
||||
data['volume_price_ratio'] = data['volume_price_ratio'].fillna(data['volume_price_ratio'].mean())
|
||||
return data
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--city', type=str, default='all')
|
||||
@@ -442,11 +481,17 @@ if __name__ == '__main__':
|
||||
else:
|
||||
cities = [city for city in args.city.split(',')]
|
||||
|
||||
db_fields = ["comm_id", "city", "region", "name", "total_score", "community_quality_score", "community_price_score",
|
||||
db_fields = ["comm_id", "city", "region", "name", 'block', 'block_rating', "address", "sqmetre_rent", "yoy",
|
||||
"rent_ro_price_ratio", "avg_price", "total_score", "community_quality_score",
|
||||
"community_price_score",
|
||||
"location_advantage_score", "flowability_score", "living_facilities_score", "education_score",
|
||||
"hospital_score", "total_score_level", "community_quality_score_level", "community_price_score_level",
|
||||
"location_advantage_score_level", "flowability_score_level", "living_facilities_score_level",
|
||||
"education_score_level", "hospital_score_level"]
|
||||
"hospital_score", "rating", "community_quality_rating", "community_price_rating",
|
||||
"location_advantage_rating", "flowability_rating", "living_facilities_rating",
|
||||
"education_rating", "hospital_rating", "supermarket_num", "park_num", "catering_num", "mall_num",
|
||||
"kindergarten_num",
|
||||
"key_primary_school_num", "ordinary_primary_school_num", "key_middle_school_num",
|
||||
"ordinary_middle_school_num", "top_three_hospital_num", "non_3a_hospital_num", "build_year", "age",
|
||||
"cubage_percent", "virescence_percent", "estate_charge", 'estate_type']
|
||||
for city in cities:
|
||||
try:
|
||||
logger.info(f"开始计算楼盘评级:{city}")
|
||||
@@ -455,40 +500,46 @@ if __name__ == '__main__':
|
||||
data["comm_id"] = data['_id']
|
||||
db_df = data[db_fields]
|
||||
df_simple = db_df.copy()
|
||||
if "business_quarter" not in df_simple.columns:
|
||||
df_simple['business_quarter'] = None
|
||||
if "block" not in df_simple.columns:
|
||||
df_simple['block'] = None
|
||||
for factor_score_field in ['total_score', 'community_quality_score', 'community_price_score',
|
||||
'location_advantage_score', 'flowability_score', 'living_facilities_score',
|
||||
'education_score', 'hospital_score']:
|
||||
key = "_"
|
||||
if factor_score_field != "total_score":
|
||||
key = f"_{factor_score_field.replace('_score', '')}"
|
||||
else:
|
||||
key = ""
|
||||
|
||||
df_simple[f'city{key}_rank'] = df_simple.groupby('city')[factor_score_field].rank(method='dense',
|
||||
ascending=False).astype(
|
||||
ascending=False).astype(
|
||||
int)
|
||||
df_simple[f'city{key}_total_rank'] = df_simple[f'city{key}_rank'].max()
|
||||
df_simple[f'region{key}_rank'] = df_simple.groupby(['city', 'region'])[factor_score_field].rank(
|
||||
method='dense', ascending=False).astype(int)
|
||||
df_simple[f'region{key}_total_rank'] = df_simple[f'region{key}_rank'].max()
|
||||
# 处理bis为None的情况 - 只对非空bis值计算排名
|
||||
df_simple[f'business_quarter{key}_total_rank'] = None
|
||||
df_simple[f'business_quarter{key}_rank'] = None
|
||||
df_simple[f'block{key}_total_rank'] = None
|
||||
df_simple[f'block{key}_rank'] = None
|
||||
# 对bis不为None的记录计算商圈排名
|
||||
bis_not_null = df_simple['business_quarter'].notna()
|
||||
bis_not_null = df_simple['block'].notna()
|
||||
if bis_not_null.any():
|
||||
df_simple.loc[bis_not_null, f'business_quarter{key}_rank'] = \
|
||||
df_simple.loc[bis_not_null].groupby(['city', 'region', 'business_quarter'])[
|
||||
factor_score_field].rank(method='dense', ascending=False).astype(int)
|
||||
df_simple.loc[bis_not_null, f'business_quarter{key}_total_rank'] = df_simple.loc[bis_not_null][
|
||||
f'business_quarter{key}_rank'].max()
|
||||
df_simple.loc[bis_not_null, f'block{key}_rank'] = \
|
||||
df_simple.loc[bis_not_null].groupby(['city', 'region', 'block'])[factor_score_field].rank(
|
||||
method='dense', ascending=False).astype(int)
|
||||
df_simple.loc[bis_not_null, f'block{key}_total_rank'] = df_simple.loc[bis_not_null][
|
||||
f'block{key}_rank'].max()
|
||||
for score_factor in ['total_score', 'community_quality_score', 'community_price_score',
|
||||
'location_advantage_score', 'flowability_score', 'living_facilities_score',
|
||||
'education_score', 'hospital_score']:
|
||||
df_simple[f'{score_factor}'] = df_simple[f'{score_factor}'].apply(lambda x: round(x * 100, 2))
|
||||
df_simple[f'{score_factor}'] = df_simple[f'{score_factor}'].apply(lambda x: round(x, 2))
|
||||
|
||||
df_simple['rent_ro_price_ratio'] = df_simple['rent_ro_price_ratio'].apply(lambda x: round(x, 2))
|
||||
df_simple['avg_price'] = df_simple['avg_price'].astype(int)
|
||||
updater = MongoDataFrameUpdater(community_rate_client)
|
||||
updater.bulk_insert_or_update_by_id(df_simple, id_fields=['comm_id'])
|
||||
logger.info(f"楼盘评级计算结束:{city}")
|
||||
except Exception as e:
|
||||
logging.error(f"{city}楼盘评级计算失败;{e}")
|
||||
print(e)
|
||||
logging.error(f"{city}楼盘评级计算失败;{e}")
|
||||
raise e
|
||||
|
||||
Reference in New Issue
Block a user