Merge remote-tracking branch 'origin/master'

This commit is contained in:
沈健
2025-06-19 09:36:13 +08:00
+26 -27
View File
@@ -1,11 +1,11 @@
import re
import logging
import argparse
import logging
import re
from datetime import datetime
import numpy as np
import math
import pandas as pd
from datetime import datetime
from goojia_utils import ElasticSearchClient
from goojia_utils import MAIN_MONGO_CLIENT, MongoDataFrameUpdater
from goojia_utils import areautils
@@ -14,16 +14,14 @@ from sklearn.mixture import GaussianMixture
from sklearn.neighbors import BallTree
from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import StandardScaler
import multiprocessing
logger = logging.getLogger(__name__)
seaweed_client = MAIN_MONGO_CLIENT.get_collection("fangjia", "seaweed")
stat_client = MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "district_stat")
city_sell_stat_client = MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "city_sell_stat")
sqmetre_rent_client = MAIN_MONGO_CLIENT.get_collection("fangjia", "sqmetre_rent")
community_rate_client = MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "community_rate")
elastic_client = ElasticSearchClient(host="192.168.0.89", username='elastic', password='elasticsearch')
# 定义楼盘评级级别数量
COMM_RATING_LEVEL_COUNT = 5
COMM_RATING_LEVEL_NAMES = {1: "S", 2: "A", 3: "B", 4: "C", 5: "D"}
@@ -210,16 +208,13 @@ def load_comms(city):
seaweed['_id'] = seaweed['_id'].apply(lambda x: str(x))
seaweed['lng'] = seaweed['lng'] / 10000000000
seaweed['lat'] = seaweed['lat'] / 10000000000
seaweed = seaweed.rename(
columns={"create_year": "build_year", "estate_type2": "estate_type", "business_quarter": "block"})
seaweed = seaweed.rename(columns={"create_year": "build_year", "estate_type2": "estate_type", "business_quarter": "block"})
year = datetime.now().year
seaweed['build_year'] = seaweed['build_year'].apply(lambda x: extract_years(x))
seaweed['age'] = seaweed['build_year'].apply(lambda x: (year - x) if pd.notna(x) else None)
seaweed = seaweed.reset_index()
for field in ['age', "cubage_percent", "virescence_percent", "estate_charge"]:
seaweed[field] = seaweed[field].apply(lambda x: str2number(x))
len(seaweed[seaweed[field].isna()])
seaweed = fill_missing_by_around(seaweed, field)
seaweed[field] = seaweed[field].fillna(seaweed[field].mean())
seaweed['age'] = seaweed['age'].apply(lambda x: int(x))
@@ -232,6 +227,7 @@ def load_comms(city):
# ## 加载小区价格数据
def load_comm_stat(city):
calc_year_month = int(dateutils.format_date(datetime.now(), "%Y%m"))
stat = pd.DataFrame(list(stat_client.find({"weekend": dateutils.get_last_weekend(), "city": city},
{"city": 1, "region": 1, "district_name": 1, "avg_price_2b": 1,
"_id": 0})))
@@ -244,6 +240,8 @@ def load_comm_stat(city):
stat = stat.merge(one_year_before_stat, on=['city', 'region', 'name'], how="left")
stat['yoy'] = (stat['avg_price'] - stat['one_year_before_avg_price']) * 100 / stat['one_year_before_avg_price']
stat['yoy'] = stat['yoy'].fillna(stat['yoy'].mean())
stat['resistance_coef'] = normalize_to_01(stat['yoy']) * 95
stat['resistance_coef'] = stat['resistance_coef'] + 2
stat['yoy'] = stat['yoy'].apply(lambda x: round(x, 2))
del stat['one_year_before_avg_price']
half_on_year_stat = pd.DataFrame(list(
@@ -266,8 +264,8 @@ def load_comm_stat(city):
{"city": 1, "region": 1, "district_name": 1, "avg_price": 1, "_id": 0})))
sqmetre_rent = sqmetre_rent.rename(columns={"district_name": "name", "avg_price": "sqmetre_rent"})
stat = stat.merge(sqmetre_rent, on=['city', 'region', 'name'], how='left')
stat['rent_ro_price_ratio'] = stat['sqmetre_rent'] * 12 / stat['avg_price'] * 100
stat['rent_ro_price_ratio'] = stat['rent_ro_price_ratio'].fillna(stat['rent_ro_price_ratio'].mean())
stat['rent_to_price_ratio'] = stat['sqmetre_rent'] * 12 / stat['avg_price'] * 100
stat['rent_to_price_ratio'] = stat['rent_to_price_ratio'].fillna(stat['rent_to_price_ratio'].mean())
return stat
@@ -314,7 +312,7 @@ factor_weights = {
'nearest_business_district_distance_score': 0.03,
'nearest_subway_station_distance_score': 0.07,
'nearest_bus_station_distance_score': 0.05,
'rent_ro_price_ratio_score': 0.1,
'rent_to_price_ratio_score': 0.1,
'volume_price_ratio_score': 0.1,
'supermarket_num_score': 0.02,
'park_num_score': 0.02,
@@ -337,7 +335,7 @@ def comms_rating(data):
"key_primary_school_num", "ordinary_primary_school_num", "key_middle_school_num",
"ordinary_middle_school_num", "top_three_hospital_num", "non_3a_hospital_num",
"nearest_business_district_distance", "nearest_subway_station_distance",
"nearest_bus_station_distance", "rent_ro_price_ratio", "volume_price_ratio"]
"nearest_bus_station_distance", "rent_to_price_ratio", "volume_price_ratio"]
for number_field in number_fields:
threshold_95 = data[number_field].quantile(0.99)
threshold_5 = data[number_field].quantile(0.01)
@@ -352,7 +350,7 @@ def comms_rating(data):
data['total_score'] = 0
factors = ["estate_type", "age", "cubage_percent", "virescence_percent", "estate_charge", "avg_price",
"sqmetre_rent", "yoy", "nearest_business_district_distance", "nearest_subway_station_distance",
"nearest_bus_station_distance", "rent_ro_price_ratio", "supermarket_num", "park_num", "catering_num",
"nearest_bus_station_distance", "rent_to_price_ratio", "supermarket_num", "park_num", "catering_num",
"mall_num", "kindergarten_num", "key_primary_school_num", "ordinary_primary_school_num",
"key_middle_school_num", "ordinary_middle_school_num", "top_three_hospital_num", "non_3a_hospital_num"]
for factor in factors:
@@ -381,7 +379,7 @@ def comms_rating(data):
f"{factor}_score"]
data['location_advantage_score'] = normalize_to_01(data['location_advantage_score'])
# 流动性打分
flowability_factors = ["rent_ro_price_ratio", "volume_price_ratio"]
flowability_factors = ["rent_to_price_ratio", "volume_price_ratio"]
data['flowability_score'] = 0
for factor in flowability_factors:
data['flowability_score'] = data[f"flowability_score"] + data[f"{factor}_score"] * factor_weights[
@@ -487,19 +485,20 @@ def process_city(city):
data["comm_id"] = data['_id']
db_fields = ["comm_id", "city", "region", "name", 'block', 'block_rating', "address", "sqmetre_rent", "yoy",
"rent_ro_price_ratio", "avg_price", "total_score", "community_quality_score",
"rent_to_price_ratio", "avg_price", "total_score", "community_quality_score",
"community_price_score", "location_advantage_score", "flowability_score", "living_facilities_score",
"education_score", "hospital_score", "rating", "community_quality_rating", "community_price_rating",
"location_advantage_rating", "flowability_rating", "living_facilities_rating", "education_rating",
"hospital_rating", "supermarket_num", "park_num", "catering_num", "mall_num", "kindergarten_num",
"key_primary_school_num", "ordinary_primary_school_num", "key_middle_school_num",
"ordinary_middle_school_num", "top_three_hospital_num", "non_3a_hospital_num", "build_year", "age",
"cubage_percent", "virescence_percent", "estate_charge", 'estate_type']
"cubage_percent", "virescence_percent", "estate_charge", 'estate_type', 'resistance_coef']
df_simple = data[db_fields].copy()
if "block" not in df_simple.columns:
df_simple['block'] = None
if "resistance_coef" not in df_simple.columns:
df_simple['resistance_coef'] = None
for factor_score_field in ['total_score', 'community_quality_score', 'community_price_score',
'location_advantage_score', 'flowability_score', 'living_facilities_score',
'education_score', 'hospital_score']:
@@ -517,8 +516,8 @@ def process_city(city):
df_simple[f'block{key}_total_rank'] = None
if df_simple['block'].notna().any():
buffer = []
data_null = data[data['block'].isna()].copy()
data_not_null = data[data['block'].notna()].copy()
data_null = df_simple[df_simple['block'].isna()].copy()
data_not_null = df_simple[df_simple['block'].notna()].copy()
buffer.append(data_null)
if len(data_not_null) > 0:
data_not_null['region_block'] = data_not_null['region'] + "_" + data_not_null['block']
@@ -528,14 +527,14 @@ def process_city(city):
group[f'block{key}_total_rank'] = group[f'block{key}_rank'].max()
buffer.append(group)
df_simple = pd.concat(buffer)
if "region_block" in df_simple.columns:
del df_simple['region_block']
for score_factor in ['total_score', 'community_quality_score', 'community_price_score','location_advantage_score', 'flowability_score', 'living_facilities_score','education_score', 'hospital_score']:
df_simple[score_factor] = df_simple[score_factor].apply(lambda x: round(x, 2))
df_simple['rent_ro_price_ratio'] = df_simple['rent_ro_price_ratio'].apply(lambda x: round(x, 2))
df_simple['avg_price'] = df_simple['avg_price'].astype(int)
df_simple['rent_to_price_ratio'] = df_simple['rent_to_price_ratio'].apply(lambda x: round(x, 2))
updater = MongoDataFrameUpdater(community_rate_client)
updater.bulk_insert_or_update_by_id(df_simple, id_fields=['comm_id'])
updater.bulk_insert_or_update_by_id(df_simple, id_fields=['comm_id'], dtype={"avg_price": int,
"build_year": int, 'resistance_coef': int})
logger.info(f"楼盘评级计算结束:{city}")
except Exception as e:
logging.error(f"{city}楼盘评级计算失败;{e}")