diff --git a/dags/stat/community/community_rate.py b/dags/stat/community/community_rate.py new file mode 100644 index 0000000..1ea99bc --- /dev/null +++ b/dags/stat/community/community_rate.py @@ -0,0 +1,455 @@ +import re +import logging +import argparse + +import numpy as np +import pandas as pd + +from datetime import datetime +from goojia_utils import ElasticSearchClient +from goojia_utils import MAIN_MONGO_CLIENT, MongoDataFrameUpdater +from goojia_utils import areautils +from goojia_utils import dateutils +from sklearn.mixture import GaussianMixture +from sklearn.neighbors import BallTree +from sklearn.preprocessing import MinMaxScaler +from sklearn.preprocessing import StandardScaler + +logger = logging.getLogger(__name__) +seaweed_client = MAIN_MONGO_CLIENT.get_collection("fangjia","seaweed") +stat_client = MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "district_stat") +sqmetre_rent_client = MAIN_MONGO_CLIENT.get_collection("fangjia", "sqmetre_rent") +community_rate_client = MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "community_rate") +elastic_client = ElasticSearchClient(host="192.168.0.89", username='elastic', password='elasticsearch') + +# 定义楼盘评级级别数量 +COMM_RATING_LEVEL_COUNT = 5 +COMM_RATING_LEVEL_NAMES = {1: "S", 2: "A", 3: "B", 4: "C", 5: "D"} + +# 定义数据预处理函数 +def extract_years(text, validate=True): + text = str(text) + # 匹配四位数且前后无其他数字 + matches = re.findall(r'(? 0: + return int(valid_years[0]) + return None + + +def fill_missing_by_around(df, field): + """ + 填充缺失的建筑年份: + 1. 使用球树(BallTree)加速空间查询 + 2. 计算1公里范围内的有效建筑年份均值 + 3. 若无有效数据则保留NaN + """ + # 转换坐标系为弧度(Haversine距离要求) + coords = np.radians(df[['lat', 'lng']].values) + + # 构建球树索引(使用Haversine距离公式) + tree = BallTree(coords, metric='haversine') + + # 定义地球半径(公里) + EARTH_RADIUS_KM = 6371 + + # 遍历每一行 + for idx, row in df.iterrows(): + if pd.isna(row[field]): + # 查询半径(将1公里转换为弧度) + radius_radians = 1 / EARTH_RADIUS_KM + + # 执行范围查询(排除自身) + neighbors = tree.query_radius( + [coords[idx]], + r=radius_radians, + return_distance=False + )[0] + neighbors = [n for n in neighbors if n != idx] # 排除当前点 + # 获取有效建筑年份 + valid_years = df.iloc[neighbors][field].dropna() + # 填充均值(若存在有效值) + if not valid_years.empty: + df.at[idx, field] = valid_years.mean() + return df + + +def str2number(text): + try: + return float(text) + except ValueError: + return None + except TypeError: + return None + + +def normalize_to_01(array): + """ + 将一维数组归一化到 [0, 1] 区间 + 参数: + - array: 输入的一维数组(列表、NumPy数组、Pandas Series) + 返回: + - 归一化后的NumPy数组(dtype=float64) + 示例: + >>> normalize_to_01([10, 20, 30]) --> array([0. , 0.5, 1. ]) + """ + # 转换为NumPy数组并确保是一维 + arr = np.array(array).reshape(-1, 1) + # 处理全相同值的情况 + if np.all(arr == arr[0]) or np.all(np.isnan(arr)): + return np.zeros_like(arr).flatten() + # 使用MinMaxScaler + x_z = StandardScaler().fit_transform(arr) + scaler = MinMaxScaler() + normalized = scaler.fit_transform(x_z) + return normalized.flatten() # 转回一维 + + +def safe_qcut_rating(values, n_bins, ascending=False): + """ + 安全的分位数评级函数,处理各种边界情况 + + Args: + values: 待分级的数值序列 + n_bins: 期望的分箱数量 + ascending: True表示数值越大评级越高,False相反 + + Returns: + 评级序列(从1开始) + """ + if len(values) == 0: + return pd.Series(dtype=int) + + unique_count = values.nunique() + + # 如果唯一值数量小于等于分箱数,直接使用排名 + if unique_count <= n_bins: + if ascending: + ratings = values.rank(method='dense', ascending=True) + else: + ratings = values.rank(method='dense', ascending=False) + return ratings.astype(int) + + # 尝试使用qcut + try: + # 使用labels=False返回数字索引,然后+1使评级从1开始 + if ascending: + # 升序:最小值得到最小评级 + ratings = pd.qcut(values, q=n_bins, labels=False, duplicates='drop') + 1 + else: + # 降序:最大值得到最小评级(等级1) + ratings = pd.qcut(values, q=n_bins, labels=False, duplicates='drop') + ratings = n_bins - ratings # 反转评级顺序 + return ratings.astype(int) + + except ValueError as e: + logging.error(f"qcut失败: {e},使用排名方法替代") + # 回退到排名方法 + if ascending: + ratings = values.rank(method='dense', ascending=True) + else: + ratings = values.rank(method='dense', ascending=False) + return ratings.astype(int) + + +def exponential_decay_weights(distances): + """ + 根据指数衰减方法计算权重。 + + 参数: + distances (array-like): 数据点与目标点之间的距离数组。 + decay_coefficient (float): 衰减系数,控制权重衰减速度。 + + 返回: + numpy.ndarray: 归一化后的权重数组。 + """ + distances = np.array(distances) + weights = 1 - np.log(distances + 1) / np.log(max(distances) + 1) + return weights + + +def gmm_train(total_rate, data, source, target): + data = data.copy() + source_values = data[source].dropna() + unique_values = source_values.nunique() + actual_components = min(total_rate, unique_values) + if unique_values <= 2: + data[target] = safe_qcut_rating(source_values, actual_components, ascending=False) + return data + + # 高斯混合模型 + gmm = GaussianMixture( + n_components=actual_components, + covariance_type='full', + max_iter=200, + n_init=10, + reg_covar=1e-6, + random_state=42 + ).fit(data[[source]]) + data = data.copy() + data.loc[:, 'cluster'] = gmm.predict(data[[source]]) + data.loc[:, 'price_mean_cluster'] = data.groupby('cluster')[source].transform('mean') + data.loc[:, target] = data['price_mean_cluster'].rank(method='dense', ascending=False).astype(int) + data = data.drop(columns=['cluster', 'price_mean_cluster'], axis=1) + return data.copy() + + +# ## 定义小区加载函数 + +def load_comms(city): + query = {"cat": "district", "status": 0, "city": city} + fields = {"city": 1, "region": 1, "name": 1, "address": 1, "lng": 1, "lat": 1, "estate_type2": 1, "create_year": 1, + "cubage_percent": 1, "virescence_percent": 1, "estate_charge": 1, "business_quarter": 1} + seaweed = pd.DataFrame(list(seaweed_client.find(query, fields))) + seaweed = seaweed.dropna(subset=['lng', 'lat']) + seaweed['_id'] = seaweed['_id'].apply(lambda x: str(x)) + seaweed['lng'] = seaweed['lng'] / 10000000000 + seaweed['lat'] = seaweed['lat'] / 10000000000 + seaweed = seaweed.rename(columns={"create_year": "build_year", "estate_type2": "estate_type"}) + seaweed['build_year'] = seaweed['build_year'].apply(lambda x: extract_years(x)) + seaweed = seaweed.reset_index() + for field in ['build_year', "cubage_percent", "virescence_percent", "estate_charge"]: + seaweed[field] = seaweed[field].apply(lambda x: str2number(x)) + na_num = len(seaweed[seaweed[field].isna()]) + seaweed = fill_missing_by_around(seaweed, field) + seaweed[field] = seaweed[field].fillna(seaweed[field].mean()) + seaweed['build_year'] = seaweed['build_year'].apply(lambda x: int(x)) + seaweed['cubage_percent'] = seaweed['cubage_percent'].apply(lambda x: round(x, 2)) + seaweed['virescence_percent'] = seaweed['virescence_percent'].apply(lambda x: round(x, 2)) + seaweed['estate_charge'] = seaweed['estate_charge'].apply(lambda x: round(x, 2)) + return seaweed + + +# ## 加载小区价格数据 + +def load_comm_stat(city): + stat = pd.DataFrame(list(stat_client.find({"weekend": dateutils.get_last_weekend(), "city": city}, + {"city": 1, "region": 1, "district_name": 1, "avg_price_2b": 1, + "_id": 0}))) + stat = stat.rename(columns={"district_name": "name", "avg_price_2b": "avg_price"}) + one_year_before_stat = pd.DataFrame(list( + stat_client.find({"weekend": dateutils.add_months(dateutils.get_last_weekend(), days=-52 * 7), "city": city}, + {"city": 1, "region": 1, "district_name": 1, "avg_price_2b": 1, "_id": 0}))) + one_year_before_stat = one_year_before_stat.rename( + columns={"district_name": "name", "avg_price_2b": "one_year_before_avg_price"}) + stat = stat.merge(one_year_before_stat, on=['city', 'region', 'name'], how="left") + stat['yoy'] = (stat['avg_price'] - stat['one_year_before_avg_price']) * 100 / stat['one_year_before_avg_price'] + stat['yoy'] = stat['yoy'].fillna(stat['yoy'].mean()) + stat['yoy'] = stat['yoy'].apply(lambda x: round(x, 2)) + del stat['one_year_before_avg_price'] + half_on_year_stat = pd.DataFrame(list( + stat_client.find({"weekend": dateutils.add_months(dateutils.get_last_weekend(), days=-26 * 7), "city": city}, + {"city": 1, "region": 1, "district_name": 1, "avg_price_2b": 1, "_id": 0}))) + half_on_year_stat = half_on_year_stat.rename( + columns={"district_name": "name", "avg_price_2b": "half_on_year_avg_price"}) + stat = stat.merge(half_on_year_stat, on=['city', 'region', 'name'], how="left") + stat['half_on_year'] = (stat['avg_price'] - stat['half_on_year_avg_price']) * 100 / stat['half_on_year_avg_price'] + stat['half_on_year'] = stat['yoy'].fillna(0) + stat['half_on_year'] = stat['yoy'].apply(lambda x: round(x, 2)) + del stat['half_on_year_avg_price'] + + firstDayOfThisMonth = dateutils.parse_date(dateutils.format_date(datetime.utcnow(), "%Y-%m"), "%Y-%m") + lastDayOfPreMonth = dateutils.add_months(firstDayOfThisMonth, days=-1) + sqmetre_rent = pd.DataFrame(list( + sqmetre_rent_client.find({"weekend": lastDayOfPreMonth, "cat": "district", "city": city}, + {"city": 1, "region": 1, "district_name": 1, "avg_price": 1, "_id": 0, + "price_to_rent": 1}))) + sqmetre_rent = sqmetre_rent.rename(columns={"district_name": "name", "avg_price": "sqmetre_rent"}) + stat = stat.merge(sqmetre_rent, on=['city', 'region', 'name'], how='left') + stat['price_to_rent'] = 1 / stat['price_to_rent'] + stat['price_to_rent'] = stat['price_to_rent'].fillna(stat['price_to_rent'].mean()) + return stat + +# ## 加载小区配套数据 + +def load_comms_summary(city): + all_community_summary = [] + for doc in elastic_client.scan_by_sql(f"select * from community_summary where city = '{city}'"): + source = doc['_source'] + community_summary = {"city": source['city'], "region": source['region'], "name": source['name']} + for key in source.keys(): + if key.endswith("num") or key.endswith("_distance"): + community_summary[key] = source[key] + all_community_summary.append(community_summary) + all_community_summary = pd.DataFrame(all_community_summary) + all_community_summary = all_community_summary.drop_duplicates(subset=['city', 'region', 'name']) + return all_community_summary + + +# ## 定义分类数据和连续性数据打分规则 + +def category_score(data, field, score_map): + data[f"{field}_score"] = data[field].apply(lambda x: score_map[x]) + return data + + +def numbic_score(data, field): + data[f"{field}_score"] = normalize_to_01(data[field]) + +estate_type_source_map = {"普通住宅": 0.5, "商住": 0, "别墅": 1} + +# ## 定义楼盘评级因子权重 + +factor_weights = { + 'estate_type_score': 0.02, + 'build_year_score': 0.02, + 'cubage_percent_score': 0.02, + 'virescence_percent_score': 0.02, + 'estate_charge_score': 0.02, + 'avg_price_score': 0.1, + 'sqmetre_rent_score': 0.05, + 'yoy_score': 0.15, + 'nearest_business_district_distance_score': 0.03, + 'nearest_subway_station_distance_score': 0.07, + 'nearest_bus_station_distance_score': 0.05, + 'price_to_rent_score': 0.1, + 'volume_price_ratio_score': 0.1, + 'supermarket_num_score': 0.02, + 'park_num_score': 0.02, + 'catering_num_score': 0.02, + 'mall_num_score': 0.04, + 'kindergarten_num_score': 0.01, + 'key_primary_school_num_score': 0.06, + 'ordinary_primary_school_num_score': 0.04, + 'key_middle_school_num_score': 0.02, + 'ordinary_middle_school_num_score': 0.02, + 'top_three_hospital_num_score': 0.03, + 'non_3a_hospital_num_score': 0.02 +} +# ## 楼盘评级打分 +def comms_rating(data): + data['build_year'] = data['build_year'].max() - data['build_year'] + number_fields = ["build_year", "cubage_percent", "virescence_percent", "estate_charge", "avg_price", "sqmetre_rent", + "yoy", "supermarket_num", "park_num", "catering_num", "mall_num", "kindergarten_num", + "key_primary_school_num", "ordinary_primary_school_num", "key_middle_school_num", + "ordinary_middle_school_num", "top_three_hospital_num", "non_3a_hospital_num", + "nearest_business_district_distance", "nearest_subway_station_distance", + "nearest_bus_station_distance", "price_to_rent", "volume_price_ratio"] + for number_field in number_fields: + threshold_95 = data[number_field].quantile(0.99) + threshold_5 = data[number_field].quantile(0.01) + # 2. 修正:将低于95分位的数据替换为 95分位值 + data[number_field] = data[number_field].apply(lambda x: threshold_95 if x >= threshold_95 else x) + data[number_field] = data[number_field].apply(lambda x: threshold_5 if x <= threshold_5 else x) + data[f'{number_field}_score'] = normalize_to_01(data[number_field]) + for category_field in ['estate_type']: + category_score(data, category_field, estate_type_source_map) + data['total_score'] = 0 + factors = ["estate_type", "build_year", "cubage_percent", "virescence_percent", "estate_charge", "avg_price", + "sqmetre_rent", "yoy", "nearest_business_district_distance", "nearest_subway_station_distance", + "nearest_bus_station_distance", "price_to_rent", "supermarket_num", "park_num", "catering_num", + "mall_num", "kindergarten_num", "key_primary_school_num", "ordinary_primary_school_num", + "key_middle_school_num", "ordinary_middle_school_num", "top_three_hospital_num", "non_3a_hospital_num"] + for factor in factors: + data['total_score'] = data[f"total_score"] + data[f"{factor}_score"] * factor_weights[f"{factor}_score"] + data['total_score'] = normalize_to_01(data['total_score']) + # 小区品质打分 + community_quality_factors = ["estate_type", "build_year", "cubage_percent", "virescence_percent", "estate_charge"] + data['community_quality_score'] = 0 + for factor in community_quality_factors: + data['community_quality_score'] = data[f"community_quality_score"] + data[f"{factor}_score"] * factor_weights[ + f"{factor}_score"] + data['community_quality_score'] = normalize_to_01(data['community_quality_score']) + # 小区价格打分 + community_price_factors = ["avg_price", "sqmetre_rent", "yoy"] + data['community_price_score'] = 0 + for factor in community_price_factors: + data['community_price_score'] = data[f"community_price_score"] + data[f"{factor}_score"] * factor_weights[ + f"{factor}_score"] + data['community_price_score'] = normalize_to_01(data['community_price_score']) + # 区位打分 + location_advantage_factors = ["nearest_business_district_distance", "nearest_subway_station_distance", + "nearest_bus_station_distance"] + data['location_advantage_score'] = 0 + for factor in location_advantage_factors: + data['location_advantage_score'] = data[f"location_advantage_score"] + data[f"{factor}_score"] * factor_weights[ + f"{factor}_score"] + data['location_advantage_score'] = normalize_to_01(data['location_advantage_score']) + # 流动性打分 + flowability_factors = ["price_to_rent", "volume_price_ratio"] + data['flowability_score'] = 0 + for factor in flowability_factors: + data['flowability_score'] = data[f"flowability_score"] + data[f"{factor}_score"] * factor_weights[ + f"{factor}_score"] + data['flowability_score'] = normalize_to_01(data['flowability_score']) + # 流动性生活配套打分 + living_facilities_factors = ["supermarket_num", "park_num", "catering_num", "mall_num"] + data['living_facilities_score'] = 0 + for factor in living_facilities_factors: + data['living_facilities_score'] = data[f"living_facilities_score"] + data[f"{factor}_score"] * factor_weights[ + f"{factor}_score"] + data['living_facilities_score'] = normalize_to_01(data['living_facilities_score']) + # 教育资源打分 + education_factors = ["kindergarten_num", "key_primary_school_num", "ordinary_primary_school_num", + "key_middle_school_num", "ordinary_middle_school_num"] + data['education_score'] = 0 + for factor in education_factors: + data['education_score'] = data[f"education_score"] + data[f"{factor}_score"] * factor_weights[f"{factor}_score"] + data['education_score'] = normalize_to_01(data['education_score']) + # 医疗资源打分 + hospital_factors = ["top_three_hospital_num", "non_3a_hospital_num"] + data['hospital_score'] = 0 + for factor in hospital_factors: + data['hospital_score'] = data[f"hospital_score"] + data[f"{factor}_score"] * factor_weights[f"{factor}_score"] + data['hospital_score'] = normalize_to_01(data['hospital_score']) + data = data.dropna(subset=['total_score']) + for score_factor in ['total_score', 'community_quality_score', 'community_price_score', 'location_advantage_score', + 'flowability_score', 'living_facilities_score', 'education_score', 'hospital_score']: + data = gmm_train(COMM_RATING_LEVEL_COUNT, data, score_factor, f'{score_factor}_level') + data[f'{score_factor}_level'] = data[f'{score_factor}_level'].map(COMM_RATING_LEVEL_NAMES) + for score_factor in ['total_score', 'community_quality_score', 'community_price_score', 'location_advantage_score', + 'flowability_score', 'living_facilities_score', 'education_score', 'hospital_score']: + data[f'{score_factor}'] = data[f'{score_factor}'].apply(lambda x: int(x * 100)) + return data + +# ## 按照城市加载楼盘评级数据宽表 + +def load_data_by_city(city): + comms = load_comms(city=city) + stat = load_comm_stat(city=city) + comms_summary = load_comms_summary(city=city) + data = comms.merge(stat, how="left", on=['city', 'region', 'name']) + data = data.merge(comms_summary, how="left", on=['city', 'region', 'name']) + num_fields = ["supermarket_num", "park_num", "catering_num", "mall_num", "kindergarten_num", + "key_primary_school_num", "ordinary_primary_school_num", "key_middle_school_num", + "ordinary_middle_school_num", "top_three_hospital_num", "non_3a_hospital_num", + "half_on_year_sell_num"] + for num_field in num_fields: + data[num_field] = data[num_field].fillna(0) + distance_fields = ["nearest_business_district_distance", "nearest_subway_station_distance", + "nearest_bus_station_distance"] + for distance_field in distance_fields: + if distance_field not in data: + data[distance_field] = 0 + else: + data[distance_field] = data[distance_field].fillna(data[distance_field].max()) + data[distance_field] = exponential_decay_weights(data[distance_field]) + data[distance_field] = data[distance_field].apply(lambda x: round(x, 6)) + data['volume_price_ratio'] = data['half_on_year_sell_num'] / data['half_on_year'].replace(0, 1e-6) + data['volume_price_ratio'] = data['volume_price_ratio'].fillna(data['volume_price_ratio'].mean()) + return data + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--city', type=str, default='all') + args = parser.parse_args() + if args.city == 'all': + cities = areautils.get_all_cities() + else: + cities = [city for city in args.city.split(',')] + for city in cities: + logger.info(f"开始计算楼盘评级:{city}") + data = load_data_by_city(city) + data = comms_rating(data) + data["comm_id"] = data['_id'] + db_fields = ["comm_id", "city", "region", "name", "total_score", "community_quality_score", "community_price_score", + "location_advantage_score", "flowability_score", "living_facilities_score", "education_score", + "hospital_score", "total_score_level", "community_quality_score_level", "community_price_score_level", + "location_advantage_score_level", "flowability_score_level", "living_facilities_score_level", + "education_score_level", "hospital_score_level"] + db_df = data[db_fields] + updater = MongoDataFrameUpdater(community_rate_client) + updater.bulk_insert_or_update_by_id(db_df, id_fields=['comm_id']) + logger.info(f"楼盘评级计算结束:{city}") \ No newline at end of file diff --git a/dags/stat/dag_community_rating.py b/dags/stat/dag_community_rating.py index b5dc78b..d81b1cb 100644 --- a/dags/stat/dag_community_rating.py +++ b/dags/stat/dag_community_rating.py @@ -6,12 +6,12 @@ with DAG( dag_id="community_rating_dag", # 每月的最后一天运行 schedule_interval="0 20 * * 6", - start_date=pendulum.datetime(2024, 7, 12, tz="Asia/Shanghai"), + start_date=pendulum.datetime(2025, 6, 12, tz="Asia/Shanghai"), description="小区楼盘评级计算", catchup=True ) as dag: t1 = BashOperator( task_id='community_rating', - bash_command='python /opt/airflow/dags/repo/dags/stat/community/rate.py' + bash_command='python /opt/airflow/dags/repo/dags/stat/community/community_rate.py --param1 "{{ dag_run.conf.get("city", "all")}}"' ) t1 \ No newline at end of file