优化楼盘评级-优化排名算法
This commit is contained in:
@@ -1,11 +1,11 @@
|
||||
import re
|
||||
import logging
|
||||
import argparse
|
||||
import logging
|
||||
import re
|
||||
from datetime import datetime
|
||||
|
||||
import numpy as np
|
||||
import math
|
||||
import pandas as pd
|
||||
|
||||
from datetime import datetime
|
||||
from goojia_utils import ElasticSearchClient
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, MongoDataFrameUpdater
|
||||
from goojia_utils import areautils
|
||||
@@ -14,16 +14,14 @@ from sklearn.mixture import GaussianMixture
|
||||
from sklearn.neighbors import BallTree
|
||||
from sklearn.preprocessing import MinMaxScaler
|
||||
from sklearn.preprocessing import StandardScaler
|
||||
import multiprocessing
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
seaweed_client = MAIN_MONGO_CLIENT.get_collection("fangjia", "seaweed")
|
||||
stat_client = MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "district_stat")
|
||||
city_sell_stat_client = MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "city_sell_stat")
|
||||
sqmetre_rent_client = MAIN_MONGO_CLIENT.get_collection("fangjia", "sqmetre_rent")
|
||||
community_rate_client = MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "community_rate")
|
||||
elastic_client = ElasticSearchClient(host="192.168.0.89", username='elastic', password='elasticsearch')
|
||||
|
||||
# 定义楼盘评级级别数量
|
||||
COMM_RATING_LEVEL_COUNT = 5
|
||||
COMM_RATING_LEVEL_NAMES = {1: "S", 2: "A", 3: "B", 4: "C", 5: "D"}
|
||||
@@ -210,16 +208,13 @@ def load_comms(city):
|
||||
seaweed['_id'] = seaweed['_id'].apply(lambda x: str(x))
|
||||
seaweed['lng'] = seaweed['lng'] / 10000000000
|
||||
seaweed['lat'] = seaweed['lat'] / 10000000000
|
||||
seaweed = seaweed.rename(
|
||||
columns={"create_year": "build_year", "estate_type2": "estate_type", "business_quarter": "block"})
|
||||
|
||||
seaweed = seaweed.rename(columns={"create_year": "build_year", "estate_type2": "estate_type", "business_quarter": "block"})
|
||||
year = datetime.now().year
|
||||
seaweed['build_year'] = seaweed['build_year'].apply(lambda x: extract_years(x))
|
||||
seaweed['age'] = seaweed['build_year'].apply(lambda x: (year - x) if pd.notna(x) else None)
|
||||
seaweed = seaweed.reset_index()
|
||||
for field in ['age', "cubage_percent", "virescence_percent", "estate_charge"]:
|
||||
seaweed[field] = seaweed[field].apply(lambda x: str2number(x))
|
||||
len(seaweed[seaweed[field].isna()])
|
||||
seaweed = fill_missing_by_around(seaweed, field)
|
||||
seaweed[field] = seaweed[field].fillna(seaweed[field].mean())
|
||||
seaweed['age'] = seaweed['age'].apply(lambda x: int(x))
|
||||
@@ -232,6 +227,7 @@ def load_comms(city):
|
||||
# ## 加载小区价格数据
|
||||
|
||||
def load_comm_stat(city):
|
||||
calc_year_month = int(dateutils.format_date(datetime.now(), "%Y%m"))
|
||||
stat = pd.DataFrame(list(stat_client.find({"weekend": dateutils.get_last_weekend(), "city": city},
|
||||
{"city": 1, "region": 1, "district_name": 1, "avg_price_2b": 1,
|
||||
"_id": 0})))
|
||||
@@ -244,6 +240,8 @@ def load_comm_stat(city):
|
||||
stat = stat.merge(one_year_before_stat, on=['city', 'region', 'name'], how="left")
|
||||
stat['yoy'] = (stat['avg_price'] - stat['one_year_before_avg_price']) * 100 / stat['one_year_before_avg_price']
|
||||
stat['yoy'] = stat['yoy'].fillna(stat['yoy'].mean())
|
||||
stat['resistance_coef'] = normalize_to_01(stat['yoy']) * 95
|
||||
stat['resistance_coef'] = stat['resistance_coef'] + 2
|
||||
stat['yoy'] = stat['yoy'].apply(lambda x: round(x, 2))
|
||||
del stat['one_year_before_avg_price']
|
||||
half_on_year_stat = pd.DataFrame(list(
|
||||
@@ -266,8 +264,8 @@ def load_comm_stat(city):
|
||||
{"city": 1, "region": 1, "district_name": 1, "avg_price": 1, "_id": 0})))
|
||||
sqmetre_rent = sqmetre_rent.rename(columns={"district_name": "name", "avg_price": "sqmetre_rent"})
|
||||
stat = stat.merge(sqmetre_rent, on=['city', 'region', 'name'], how='left')
|
||||
stat['rent_ro_price_ratio'] = stat['sqmetre_rent'] * 12 / stat['avg_price'] * 100
|
||||
stat['rent_ro_price_ratio'] = stat['rent_ro_price_ratio'].fillna(stat['rent_ro_price_ratio'].mean())
|
||||
stat['rent_to_price_ratio'] = stat['sqmetre_rent'] * 12 / stat['avg_price'] * 100
|
||||
stat['rent_to_price_ratio'] = stat['rent_to_price_ratio'].fillna(stat['rent_to_price_ratio'].mean())
|
||||
return stat
|
||||
|
||||
|
||||
@@ -314,7 +312,7 @@ factor_weights = {
|
||||
'nearest_business_district_distance_score': 0.03,
|
||||
'nearest_subway_station_distance_score': 0.07,
|
||||
'nearest_bus_station_distance_score': 0.05,
|
||||
'rent_ro_price_ratio_score': 0.1,
|
||||
'rent_to_price_ratio_score': 0.1,
|
||||
'volume_price_ratio_score': 0.1,
|
||||
'supermarket_num_score': 0.02,
|
||||
'park_num_score': 0.02,
|
||||
@@ -337,7 +335,7 @@ def comms_rating(data):
|
||||
"key_primary_school_num", "ordinary_primary_school_num", "key_middle_school_num",
|
||||
"ordinary_middle_school_num", "top_three_hospital_num", "non_3a_hospital_num",
|
||||
"nearest_business_district_distance", "nearest_subway_station_distance",
|
||||
"nearest_bus_station_distance", "rent_ro_price_ratio", "volume_price_ratio"]
|
||||
"nearest_bus_station_distance", "rent_to_price_ratio", "volume_price_ratio"]
|
||||
for number_field in number_fields:
|
||||
threshold_95 = data[number_field].quantile(0.99)
|
||||
threshold_5 = data[number_field].quantile(0.01)
|
||||
@@ -352,7 +350,7 @@ def comms_rating(data):
|
||||
data['total_score'] = 0
|
||||
factors = ["estate_type", "age", "cubage_percent", "virescence_percent", "estate_charge", "avg_price",
|
||||
"sqmetre_rent", "yoy", "nearest_business_district_distance", "nearest_subway_station_distance",
|
||||
"nearest_bus_station_distance", "rent_ro_price_ratio", "supermarket_num", "park_num", "catering_num",
|
||||
"nearest_bus_station_distance", "rent_to_price_ratio", "supermarket_num", "park_num", "catering_num",
|
||||
"mall_num", "kindergarten_num", "key_primary_school_num", "ordinary_primary_school_num",
|
||||
"key_middle_school_num", "ordinary_middle_school_num", "top_three_hospital_num", "non_3a_hospital_num"]
|
||||
for factor in factors:
|
||||
@@ -381,7 +379,7 @@ def comms_rating(data):
|
||||
f"{factor}_score"]
|
||||
data['location_advantage_score'] = normalize_to_01(data['location_advantage_score'])
|
||||
# 流动性打分
|
||||
flowability_factors = ["rent_ro_price_ratio", "volume_price_ratio"]
|
||||
flowability_factors = ["rent_to_price_ratio", "volume_price_ratio"]
|
||||
data['flowability_score'] = 0
|
||||
for factor in flowability_factors:
|
||||
data['flowability_score'] = data[f"flowability_score"] + data[f"{factor}_score"] * factor_weights[
|
||||
@@ -487,19 +485,20 @@ def process_city(city):
|
||||
data["comm_id"] = data['_id']
|
||||
|
||||
db_fields = ["comm_id", "city", "region", "name", 'block', 'block_rating', "address", "sqmetre_rent", "yoy",
|
||||
"rent_ro_price_ratio", "avg_price", "total_score", "community_quality_score",
|
||||
"rent_to_price_ratio", "avg_price", "total_score", "community_quality_score",
|
||||
"community_price_score", "location_advantage_score", "flowability_score", "living_facilities_score",
|
||||
"education_score", "hospital_score", "rating", "community_quality_rating", "community_price_rating",
|
||||
"location_advantage_rating", "flowability_rating", "living_facilities_rating", "education_rating",
|
||||
"hospital_rating", "supermarket_num", "park_num", "catering_num", "mall_num", "kindergarten_num",
|
||||
"key_primary_school_num", "ordinary_primary_school_num", "key_middle_school_num",
|
||||
"ordinary_middle_school_num", "top_three_hospital_num", "non_3a_hospital_num", "build_year", "age",
|
||||
"cubage_percent", "virescence_percent", "estate_charge", 'estate_type']
|
||||
"cubage_percent", "virescence_percent", "estate_charge", 'estate_type', 'resistance_coef']
|
||||
|
||||
df_simple = data[db_fields].copy()
|
||||
if "block" not in df_simple.columns:
|
||||
df_simple['block'] = None
|
||||
|
||||
if "resistance_coef" not in df_simple.columns:
|
||||
df_simple['resistance_coef'] = None
|
||||
for factor_score_field in ['total_score', 'community_quality_score', 'community_price_score',
|
||||
'location_advantage_score', 'flowability_score', 'living_facilities_score',
|
||||
'education_score', 'hospital_score']:
|
||||
@@ -517,8 +516,8 @@ def process_city(city):
|
||||
df_simple[f'block{key}_total_rank'] = None
|
||||
if df_simple['block'].notna().any():
|
||||
buffer = []
|
||||
data_null = data[data['block'].isna()].copy()
|
||||
data_not_null = data[data['block'].notna()].copy()
|
||||
data_null = df_simple[df_simple['block'].isna()].copy()
|
||||
data_not_null = df_simple[df_simple['block'].notna()].copy()
|
||||
buffer.append(data_null)
|
||||
if len(data_not_null) > 0:
|
||||
data_not_null['region_block'] = data_not_null['region'] + "_" + data_not_null['block']
|
||||
@@ -528,14 +527,14 @@ def process_city(city):
|
||||
group[f'block{key}_total_rank'] = group[f'block{key}_rank'].max()
|
||||
buffer.append(group)
|
||||
df_simple = pd.concat(buffer)
|
||||
if "region_block" in df_simple.columns:
|
||||
del df_simple['region_block']
|
||||
for score_factor in ['total_score', 'community_quality_score', 'community_price_score','location_advantage_score', 'flowability_score', 'living_facilities_score','education_score', 'hospital_score']:
|
||||
df_simple[score_factor] = df_simple[score_factor].apply(lambda x: round(x, 2))
|
||||
|
||||
df_simple['rent_ro_price_ratio'] = df_simple['rent_ro_price_ratio'].apply(lambda x: round(x, 2))
|
||||
df_simple['avg_price'] = df_simple['avg_price'].astype(int)
|
||||
|
||||
df_simple['rent_to_price_ratio'] = df_simple['rent_to_price_ratio'].apply(lambda x: round(x, 2))
|
||||
updater = MongoDataFrameUpdater(community_rate_client)
|
||||
updater.bulk_insert_or_update_by_id(df_simple, id_fields=['comm_id'])
|
||||
updater.bulk_insert_or_update_by_id(df_simple, id_fields=['comm_id'], dtype={"avg_price": int,
|
||||
"build_year": int, 'resistance_coef': int})
|
||||
logger.info(f"楼盘评级计算结束:{city}")
|
||||
except Exception as e:
|
||||
logging.error(f"{city}楼盘评级计算失败;{e}")
|
||||
|
||||
Reference in New Issue
Block a user