优化楼盘评级算法

This commit is contained in:
2025-07-03 16:03:07 +08:00
parent 5cd3470091
commit 047b826691
2 changed files with 50 additions and 33 deletions
+49 -32
View File
@@ -1,10 +1,10 @@
import argparse
import logging
import os
import re
from datetime import datetime
import numpy as np
import math
import pandas as pd
from goojia_utils import ElasticSearchClient
from goojia_utils import MAIN_MONGO_CLIENT, MongoDataFrameUpdater
@@ -14,10 +14,16 @@ from sklearn.mixture import GaussianMixture
from sklearn.neighbors import BallTree
from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import StandardScaler
from sqlalchemy import create_engine
os.environ["NLS_LANG"] = "AMERICAN_AMERICA.AL32UTF8"
engine236 = create_engine("postgresql+psycopg2://postgres:Goojia123456@192.168.0.235:5433/fangjia", echo=False)
logger = logging.getLogger(__name__)
seaweed_client = MAIN_MONGO_CLIENT.get_collection("fangjia", "seaweed")
stat_client = MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "district_stat")
stat_client = MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "district_sell_stat")
city_sell_stat_client = MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "city_sell_stat")
sqmetre_rent_client = MAIN_MONGO_CLIENT.get_collection("fangjia", "sqmetre_rent")
community_rate_client = MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "community_rate")
@@ -228,18 +234,17 @@ def load_comms(city):
# ## 加载小区价格数据
def load_comm_stat(city):
calc_year_month = int(dateutils.format_date(datetime.now(), "%Y%m"))
stat = pd.DataFrame(list(stat_client.find({"weekend": dateutils.get_last_weekend(), "city": city},
{"city": 1, "region": 1, "district_name": 1, "avg_price_2b": 1,
"_id": 0})))
stat = stat.rename(columns={"district_name": "name", "avg_price_2b": "avg_price"})
def load_comm_stat(city, calc_year_month):
one_year_ago = dateutils.add_months(dateutils.parse_date(str(calc_year_month),"%Y%m"), years=-1)
half_on_year = dateutils.add_months(dateutils.parse_date(str(calc_year_month),"%Y%m"), months=-6)
one_year_ago_year_month = int(dateutils.format_date(one_year_ago, "%Y%m"))
half_on_year_year_month = int(dateutils.format_date(half_on_year, "%Y%m"))
stat = pd.DataFrame(list(stat_client.find({"date": calc_year_month, "city": city}, {"city": 1, "region": 1, "district_name": 1, "avg_price": 1, "_id": 0})))
stat = stat.rename(columns={"district_name": "name"})
stat['org_avg_price'] = stat["avg_price"]
one_year_before_stat = pd.DataFrame(list(
stat_client.find({"weekend": dateutils.add_months(dateutils.get_last_weekend(), days=-52 * 7), "city": city},
{"city": 1, "region": 1, "district_name": 1, "avg_price_2b": 1, "_id": 0})))
one_year_before_stat = one_year_before_stat.rename(
columns={"district_name": "name", "avg_price_2b": "one_year_before_avg_price"})
one_year_before_stat = pd.DataFrame(list(stat_client.find({"date": one_year_ago_year_month, "city": city}, {"city": 1, "region": 1, "district_name": 1, "avg_price": 1, "_id": 0})))
one_year_before_stat = one_year_before_stat.rename(columns={"district_name": "name", "avg_price": "one_year_before_avg_price"})
stat = stat.merge(one_year_before_stat, on=['city', 'region', 'name'], how="left")
stat['yoy'] = (stat['avg_price'] - stat['one_year_before_avg_price']) * 100 / stat['one_year_before_avg_price']
stat['org_yoy'] = stat["yoy"]
@@ -248,11 +253,8 @@ def load_comm_stat(city):
stat['resistance_coef'] = stat['resistance_coef'] + 2
stat['yoy'] = stat['yoy'].apply(lambda x: round(x, 2))
del stat['one_year_before_avg_price']
half_on_year_stat = pd.DataFrame(list(
stat_client.find({"weekend": dateutils.add_months(dateutils.get_last_weekend(), days=-26 * 7), "city": city},
{"city": 1, "region": 1, "district_name": 1, "avg_price_2b": 1, "_id": 0})))
half_on_year_stat = half_on_year_stat.rename(
columns={"district_name": "name", "avg_price_2b": "half_on_year_avg_price"})
half_on_year_stat = pd.DataFrame(list(stat_client.find({"date": half_on_year_year_month, "city": city}, {"city": 1, "region": 1, "district_name": 1, "avg_price": 1, "_id": 0})))
half_on_year_stat = half_on_year_stat.rename(columns={"district_name": "name", "avg_price": "half_on_year_avg_price"})
if half_on_year_stat.empty:
return pd.DataFrame()
stat = stat.merge(half_on_year_stat, on=['city', 'region', 'name'], how="left")
@@ -260,11 +262,7 @@ def load_comm_stat(city):
stat['half_on_year'] = stat['yoy'].fillna(0)
stat['half_on_year'] = stat['yoy'].apply(lambda x: round(x, 2))
del stat['half_on_year_avg_price']
firstDayOfThisMonth = dateutils.parse_date(dateutils.format_date(datetime.utcnow(), "%Y-%m"), "%Y-%m")
lastDayOfPreMonth = dateutils.add_months(firstDayOfThisMonth, days=-1)
sqmetre_rent = pd.DataFrame(list(sqmetre_rent_client.find({"weekend": lastDayOfPreMonth, "cat": "district", "city": city},
{"city": 1, "region": 1, "district_name": 1, "avg_price": 1, "_id": 0})))
sqmetre_rent = pd.DataFrame(list(sqmetre_rent_client.find({"s_date": calc_year_month, "cat": "district", "city": city},{"city": 1, "region": 1, "district_name": 1, "avg_price": 1, "_id": 0})))
sqmetre_rent = sqmetre_rent.rename(columns={"district_name": "name", "avg_price": "sqmetre_rent"})
stat = stat.merge(sqmetre_rent, on=['city', 'region', 'name'], how='left')
stat['rent_to_price_ratio'] = stat['sqmetre_rent'] * 12 / stat['avg_price'] * 100
@@ -452,13 +450,25 @@ def comms_rating(data):
data[f'{score_factor}'] = data[f'{score_factor}'].apply(lambda x: round(x * 100, 6))
return data
city_code_map = {"深圳": '440300', "临沂" : "371300"}
def load_house_data(city, calc_year_month):
end_year_month = int(dateutils.format_date(dateutils.add_months(dateutils.parse_date(str(calc_year_month), "%Y%m"), months=1), "%Y%m"))
half_on_year = dateutils.add_months(dateutils.parse_date(str(calc_year_month), "%Y%m"), months=-6)
half_on_year_year_month = int(dateutils.format_date(half_on_year, "%Y%m"))
sql = f"""
select gujia_comm_id as comm_id, count(*) as half_on_year from base.bigdata_forsale where urbn_id = '{city_code_map[city]}'
and data_crt_tm >= to_date('{half_on_year_year_month}', 'YYYYMM') and data_crt_tm < to_date('{end_year_month}', 'YYYYMM')
group by gujia_comm_id ;
"""
return pd.read_sql(sql, con=engine236)
# ## 按照城市加载楼盘评级数据宽表
def load_data_by_city(city):
def load_data_by_city(city, calc_year_month):
comms = load_comms(city=city)
stat = load_comm_stat(city=city)
stat = load_comm_stat(city, calc_year_month)
comms_summary = load_comms_summary(city=city)
house_stat = load_house_data(city, calc_year_month)
data = comms.merge(stat, how="left", on=['city', 'region', 'name'])
data = data.merge(comms_summary, how="left", on=['comm_id'])
num_fields = ["supermarket_num", "park_num", "catering_num", "mall_num", "kindergarten_num",
@@ -482,10 +492,12 @@ def load_data_by_city(city):
import multiprocessing
def process_city(city):
def process_city(args):
city = args['city']
calc_year_month = args['calc_year_month']
try:
logger.info(f"开始计算楼盘评级:{city}")
data = load_data_by_city(city)
logger.info(f"开始计算楼盘评级:{city} 年月:{calc_year_month}")
data = load_data_by_city(city, calc_year_month)
data = comms_rating(data)
db_fields = ["comm_id", "city", "region", "name", 'block', 'block_rating', "address", "sqmetre_rent", "yoy",
"rent_to_price_ratio", "avg_price", "total_score", "community_quality_score",
@@ -546,7 +558,8 @@ def process_city(city):
for field in df_simple.columns:
if "_rank" in field or "_num" in field:
dtype[field] = int
updater.bulk_insert_or_update_by_id(df_simple, id_fields=['comm_id'], dtype=dtype)
df_simple['date'] = calc_year_month
updater.bulk_insert_or_update_by_id(df_simple, id_fields=['comm_id', 'date'], dtype=dtype)
logger.info(f"楼盘评级计算结束:{city}")
except Exception as e:
logging.error(f"{city}楼盘评级计算失败;{e}")
@@ -555,12 +568,16 @@ def process_city(city):
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--city', type=str, default='all')
parser.add_argument('--month', type=str, default='')
args = parser.parse_args()
if args.city == 'all':
cities = areautils.get_all_cities()
else:
cities = [city.strip() for city in args.city.split(',') if city.strip()]
if args.month == '':
calc_year_month = int(dateutils.format_date(dateutils.add_months(datetime.now(), months=-1), "%Y%m"))
else:
calc_year_month = int(args.month)
args_list = [{"calc_year_month": calc_year_month, "city": city} for city in cities]
with multiprocessing.Pool(processes=min(8, len(cities))) as pool:
pool.map(process_city, cities)
pool.map(process_city, args_list)
+1 -1
View File
@@ -12,6 +12,6 @@ with DAG(
) as dag:
t1 = BashOperator(
task_id='community_rating',
bash_command='python /opt/airflow/dags/repo/dags/stat/community/community_rate.py --city "{{ dag_run.conf.get("city", "all")}}"'
bash_command='python /opt/airflow/dags/repo/dags/stat/community/community_rate.py --city "{{ dag_run.conf.get("city", "all")}}" --month "{{ dag_run.conf.get("month", "")}}"'
)
t1