diff --git a/dags/stat/community/community_rate.py b/dags/stat/community/community_rate.py index c985ddf..8cde798 100644 --- a/dags/stat/community/community_rate.py +++ b/dags/stat/community/community_rate.py @@ -1,11 +1,11 @@ -import re -import logging import argparse +import logging +import re +from datetime import datetime import numpy as np +import math import pandas as pd - -from datetime import datetime from goojia_utils import ElasticSearchClient from goojia_utils import MAIN_MONGO_CLIENT, MongoDataFrameUpdater from goojia_utils import areautils @@ -14,16 +14,14 @@ from sklearn.mixture import GaussianMixture from sklearn.neighbors import BallTree from sklearn.preprocessing import MinMaxScaler from sklearn.preprocessing import StandardScaler -import multiprocessing - logger = logging.getLogger(__name__) seaweed_client = MAIN_MONGO_CLIENT.get_collection("fangjia", "seaweed") stat_client = MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "district_stat") +city_sell_stat_client = MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "city_sell_stat") sqmetre_rent_client = MAIN_MONGO_CLIENT.get_collection("fangjia", "sqmetre_rent") community_rate_client = MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "community_rate") elastic_client = ElasticSearchClient(host="192.168.0.89", username='elastic', password='elasticsearch') - # 定义楼盘评级级别数量 COMM_RATING_LEVEL_COUNT = 5 COMM_RATING_LEVEL_NAMES = {1: "S", 2: "A", 3: "B", 4: "C", 5: "D"} @@ -210,16 +208,13 @@ def load_comms(city): seaweed['_id'] = seaweed['_id'].apply(lambda x: str(x)) seaweed['lng'] = seaweed['lng'] / 10000000000 seaweed['lat'] = seaweed['lat'] / 10000000000 - seaweed = seaweed.rename( - columns={"create_year": "build_year", "estate_type2": "estate_type", "business_quarter": "block"}) - + seaweed = seaweed.rename(columns={"create_year": "build_year", "estate_type2": "estate_type", "business_quarter": "block"}) year = datetime.now().year seaweed['build_year'] = seaweed['build_year'].apply(lambda x: extract_years(x)) seaweed['age'] = seaweed['build_year'].apply(lambda x: (year - x) if pd.notna(x) else None) seaweed = seaweed.reset_index() for field in ['age', "cubage_percent", "virescence_percent", "estate_charge"]: seaweed[field] = seaweed[field].apply(lambda x: str2number(x)) - len(seaweed[seaweed[field].isna()]) seaweed = fill_missing_by_around(seaweed, field) seaweed[field] = seaweed[field].fillna(seaweed[field].mean()) seaweed['age'] = seaweed['age'].apply(lambda x: int(x)) @@ -232,6 +227,7 @@ def load_comms(city): # ## 加载小区价格数据 def load_comm_stat(city): + calc_year_month = int(dateutils.format_date(datetime.now(), "%Y%m")) stat = pd.DataFrame(list(stat_client.find({"weekend": dateutils.get_last_weekend(), "city": city}, {"city": 1, "region": 1, "district_name": 1, "avg_price_2b": 1, "_id": 0}))) @@ -244,6 +240,8 @@ def load_comm_stat(city): stat = stat.merge(one_year_before_stat, on=['city', 'region', 'name'], how="left") stat['yoy'] = (stat['avg_price'] - stat['one_year_before_avg_price']) * 100 / stat['one_year_before_avg_price'] stat['yoy'] = stat['yoy'].fillna(stat['yoy'].mean()) + stat['resistance_coef'] = normalize_to_01(stat['yoy']) * 95 + stat['resistance_coef'] = stat['resistance_coef'] + 2 stat['yoy'] = stat['yoy'].apply(lambda x: round(x, 2)) del stat['one_year_before_avg_price'] half_on_year_stat = pd.DataFrame(list( @@ -266,8 +264,8 @@ def load_comm_stat(city): {"city": 1, "region": 1, "district_name": 1, "avg_price": 1, "_id": 0}))) sqmetre_rent = sqmetre_rent.rename(columns={"district_name": "name", "avg_price": "sqmetre_rent"}) stat = stat.merge(sqmetre_rent, on=['city', 'region', 'name'], how='left') - stat['rent_ro_price_ratio'] = stat['sqmetre_rent'] * 12 / stat['avg_price'] * 100 - stat['rent_ro_price_ratio'] = stat['rent_ro_price_ratio'].fillna(stat['rent_ro_price_ratio'].mean()) + stat['rent_to_price_ratio'] = stat['sqmetre_rent'] * 12 / stat['avg_price'] * 100 + stat['rent_to_price_ratio'] = stat['rent_to_price_ratio'].fillna(stat['rent_to_price_ratio'].mean()) return stat @@ -314,7 +312,7 @@ factor_weights = { 'nearest_business_district_distance_score': 0.03, 'nearest_subway_station_distance_score': 0.07, 'nearest_bus_station_distance_score': 0.05, - 'rent_ro_price_ratio_score': 0.1, + 'rent_to_price_ratio_score': 0.1, 'volume_price_ratio_score': 0.1, 'supermarket_num_score': 0.02, 'park_num_score': 0.02, @@ -337,7 +335,7 @@ def comms_rating(data): "key_primary_school_num", "ordinary_primary_school_num", "key_middle_school_num", "ordinary_middle_school_num", "top_three_hospital_num", "non_3a_hospital_num", "nearest_business_district_distance", "nearest_subway_station_distance", - "nearest_bus_station_distance", "rent_ro_price_ratio", "volume_price_ratio"] + "nearest_bus_station_distance", "rent_to_price_ratio", "volume_price_ratio"] for number_field in number_fields: threshold_95 = data[number_field].quantile(0.99) threshold_5 = data[number_field].quantile(0.01) @@ -352,7 +350,7 @@ def comms_rating(data): data['total_score'] = 0 factors = ["estate_type", "age", "cubage_percent", "virescence_percent", "estate_charge", "avg_price", "sqmetre_rent", "yoy", "nearest_business_district_distance", "nearest_subway_station_distance", - "nearest_bus_station_distance", "rent_ro_price_ratio", "supermarket_num", "park_num", "catering_num", + "nearest_bus_station_distance", "rent_to_price_ratio", "supermarket_num", "park_num", "catering_num", "mall_num", "kindergarten_num", "key_primary_school_num", "ordinary_primary_school_num", "key_middle_school_num", "ordinary_middle_school_num", "top_three_hospital_num", "non_3a_hospital_num"] for factor in factors: @@ -381,7 +379,7 @@ def comms_rating(data): f"{factor}_score"] data['location_advantage_score'] = normalize_to_01(data['location_advantage_score']) # 流动性打分 - flowability_factors = ["rent_ro_price_ratio", "volume_price_ratio"] + flowability_factors = ["rent_to_price_ratio", "volume_price_ratio"] data['flowability_score'] = 0 for factor in flowability_factors: data['flowability_score'] = data[f"flowability_score"] + data[f"{factor}_score"] * factor_weights[ @@ -487,19 +485,20 @@ def process_city(city): data["comm_id"] = data['_id'] db_fields = ["comm_id", "city", "region", "name", 'block', 'block_rating', "address", "sqmetre_rent", "yoy", - "rent_ro_price_ratio", "avg_price", "total_score", "community_quality_score", + "rent_to_price_ratio", "avg_price", "total_score", "community_quality_score", "community_price_score", "location_advantage_score", "flowability_score", "living_facilities_score", "education_score", "hospital_score", "rating", "community_quality_rating", "community_price_rating", "location_advantage_rating", "flowability_rating", "living_facilities_rating", "education_rating", "hospital_rating", "supermarket_num", "park_num", "catering_num", "mall_num", "kindergarten_num", "key_primary_school_num", "ordinary_primary_school_num", "key_middle_school_num", "ordinary_middle_school_num", "top_three_hospital_num", "non_3a_hospital_num", "build_year", "age", - "cubage_percent", "virescence_percent", "estate_charge", 'estate_type'] + "cubage_percent", "virescence_percent", "estate_charge", 'estate_type', 'resistance_coef'] df_simple = data[db_fields].copy() if "block" not in df_simple.columns: df_simple['block'] = None - + if "resistance_coef" not in df_simple.columns: + df_simple['resistance_coef'] = None for factor_score_field in ['total_score', 'community_quality_score', 'community_price_score', 'location_advantage_score', 'flowability_score', 'living_facilities_score', 'education_score', 'hospital_score']: @@ -517,8 +516,8 @@ def process_city(city): df_simple[f'block{key}_total_rank'] = None if df_simple['block'].notna().any(): buffer = [] - data_null = data[data['block'].isna()].copy() - data_not_null = data[data['block'].notna()].copy() + data_null = df_simple[df_simple['block'].isna()].copy() + data_not_null = df_simple[df_simple['block'].notna()].copy() buffer.append(data_null) if len(data_not_null) > 0: data_not_null['region_block'] = data_not_null['region'] + "_" + data_not_null['block'] @@ -528,14 +527,14 @@ def process_city(city): group[f'block{key}_total_rank'] = group[f'block{key}_rank'].max() buffer.append(group) df_simple = pd.concat(buffer) + if "region_block" in df_simple.columns: + del df_simple['region_block'] for score_factor in ['total_score', 'community_quality_score', 'community_price_score','location_advantage_score', 'flowability_score', 'living_facilities_score','education_score', 'hospital_score']: df_simple[score_factor] = df_simple[score_factor].apply(lambda x: round(x, 2)) - - df_simple['rent_ro_price_ratio'] = df_simple['rent_ro_price_ratio'].apply(lambda x: round(x, 2)) - df_simple['avg_price'] = df_simple['avg_price'].astype(int) - + df_simple['rent_to_price_ratio'] = df_simple['rent_to_price_ratio'].apply(lambda x: round(x, 2)) updater = MongoDataFrameUpdater(community_rate_client) - updater.bulk_insert_or_update_by_id(df_simple, id_fields=['comm_id']) + updater.bulk_insert_or_update_by_id(df_simple, id_fields=['comm_id'], dtype={"avg_price": int, + "build_year": int, 'resistance_coef': int}) logger.info(f"楼盘评级计算结束:{city}") except Exception as e: logging.error(f"{city}楼盘评级计算失败;{e}")