优化楼盘评级算法
This commit is contained in:
@@ -1,10 +1,10 @@
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from datetime import datetime
|
||||
|
||||
import numpy as np
|
||||
import math
|
||||
import pandas as pd
|
||||
from goojia_utils import ElasticSearchClient
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, MongoDataFrameUpdater
|
||||
@@ -14,10 +14,16 @@ from sklearn.mixture import GaussianMixture
|
||||
from sklearn.neighbors import BallTree
|
||||
from sklearn.preprocessing import MinMaxScaler
|
||||
from sklearn.preprocessing import StandardScaler
|
||||
from sqlalchemy import create_engine
|
||||
|
||||
os.environ["NLS_LANG"] = "AMERICAN_AMERICA.AL32UTF8"
|
||||
engine236 = create_engine("postgresql+psycopg2://postgres:Goojia123456@192.168.0.235:5433/fangjia", echo=False)
|
||||
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
seaweed_client = MAIN_MONGO_CLIENT.get_collection("fangjia", "seaweed")
|
||||
stat_client = MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "district_stat")
|
||||
stat_client = MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "district_sell_stat")
|
||||
city_sell_stat_client = MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "city_sell_stat")
|
||||
sqmetre_rent_client = MAIN_MONGO_CLIENT.get_collection("fangjia", "sqmetre_rent")
|
||||
community_rate_client = MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "community_rate")
|
||||
@@ -228,18 +234,17 @@ def load_comms(city):
|
||||
|
||||
# ## 加载小区价格数据
|
||||
|
||||
def load_comm_stat(city):
|
||||
calc_year_month = int(dateutils.format_date(datetime.now(), "%Y%m"))
|
||||
stat = pd.DataFrame(list(stat_client.find({"weekend": dateutils.get_last_weekend(), "city": city},
|
||||
{"city": 1, "region": 1, "district_name": 1, "avg_price_2b": 1,
|
||||
"_id": 0})))
|
||||
stat = stat.rename(columns={"district_name": "name", "avg_price_2b": "avg_price"})
|
||||
def load_comm_stat(city, calc_year_month):
|
||||
one_year_ago = dateutils.add_months(dateutils.parse_date(str(calc_year_month),"%Y%m"), years=-1)
|
||||
half_on_year = dateutils.add_months(dateutils.parse_date(str(calc_year_month),"%Y%m"), months=-6)
|
||||
one_year_ago_year_month = int(dateutils.format_date(one_year_ago, "%Y%m"))
|
||||
half_on_year_year_month = int(dateutils.format_date(half_on_year, "%Y%m"))
|
||||
|
||||
stat = pd.DataFrame(list(stat_client.find({"date": calc_year_month, "city": city}, {"city": 1, "region": 1, "district_name": 1, "avg_price": 1, "_id": 0})))
|
||||
stat = stat.rename(columns={"district_name": "name"})
|
||||
stat['org_avg_price'] = stat["avg_price"]
|
||||
one_year_before_stat = pd.DataFrame(list(
|
||||
stat_client.find({"weekend": dateutils.add_months(dateutils.get_last_weekend(), days=-52 * 7), "city": city},
|
||||
{"city": 1, "region": 1, "district_name": 1, "avg_price_2b": 1, "_id": 0})))
|
||||
one_year_before_stat = one_year_before_stat.rename(
|
||||
columns={"district_name": "name", "avg_price_2b": "one_year_before_avg_price"})
|
||||
one_year_before_stat = pd.DataFrame(list(stat_client.find({"date": one_year_ago_year_month, "city": city}, {"city": 1, "region": 1, "district_name": 1, "avg_price": 1, "_id": 0})))
|
||||
one_year_before_stat = one_year_before_stat.rename(columns={"district_name": "name", "avg_price": "one_year_before_avg_price"})
|
||||
stat = stat.merge(one_year_before_stat, on=['city', 'region', 'name'], how="left")
|
||||
stat['yoy'] = (stat['avg_price'] - stat['one_year_before_avg_price']) * 100 / stat['one_year_before_avg_price']
|
||||
stat['org_yoy'] = stat["yoy"]
|
||||
@@ -248,11 +253,8 @@ def load_comm_stat(city):
|
||||
stat['resistance_coef'] = stat['resistance_coef'] + 2
|
||||
stat['yoy'] = stat['yoy'].apply(lambda x: round(x, 2))
|
||||
del stat['one_year_before_avg_price']
|
||||
half_on_year_stat = pd.DataFrame(list(
|
||||
stat_client.find({"weekend": dateutils.add_months(dateutils.get_last_weekend(), days=-26 * 7), "city": city},
|
||||
{"city": 1, "region": 1, "district_name": 1, "avg_price_2b": 1, "_id": 0})))
|
||||
half_on_year_stat = half_on_year_stat.rename(
|
||||
columns={"district_name": "name", "avg_price_2b": "half_on_year_avg_price"})
|
||||
half_on_year_stat = pd.DataFrame(list(stat_client.find({"date": half_on_year_year_month, "city": city}, {"city": 1, "region": 1, "district_name": 1, "avg_price": 1, "_id": 0})))
|
||||
half_on_year_stat = half_on_year_stat.rename(columns={"district_name": "name", "avg_price": "half_on_year_avg_price"})
|
||||
if half_on_year_stat.empty:
|
||||
return pd.DataFrame()
|
||||
stat = stat.merge(half_on_year_stat, on=['city', 'region', 'name'], how="left")
|
||||
@@ -260,11 +262,7 @@ def load_comm_stat(city):
|
||||
stat['half_on_year'] = stat['yoy'].fillna(0)
|
||||
stat['half_on_year'] = stat['yoy'].apply(lambda x: round(x, 2))
|
||||
del stat['half_on_year_avg_price']
|
||||
|
||||
firstDayOfThisMonth = dateutils.parse_date(dateutils.format_date(datetime.utcnow(), "%Y-%m"), "%Y-%m")
|
||||
lastDayOfPreMonth = dateutils.add_months(firstDayOfThisMonth, days=-1)
|
||||
sqmetre_rent = pd.DataFrame(list(sqmetre_rent_client.find({"weekend": lastDayOfPreMonth, "cat": "district", "city": city},
|
||||
{"city": 1, "region": 1, "district_name": 1, "avg_price": 1, "_id": 0})))
|
||||
sqmetre_rent = pd.DataFrame(list(sqmetre_rent_client.find({"s_date": calc_year_month, "cat": "district", "city": city},{"city": 1, "region": 1, "district_name": 1, "avg_price": 1, "_id": 0})))
|
||||
sqmetre_rent = sqmetre_rent.rename(columns={"district_name": "name", "avg_price": "sqmetre_rent"})
|
||||
stat = stat.merge(sqmetre_rent, on=['city', 'region', 'name'], how='left')
|
||||
stat['rent_to_price_ratio'] = stat['sqmetre_rent'] * 12 / stat['avg_price'] * 100
|
||||
@@ -452,13 +450,25 @@ def comms_rating(data):
|
||||
data[f'{score_factor}'] = data[f'{score_factor}'].apply(lambda x: round(x * 100, 6))
|
||||
return data
|
||||
|
||||
city_code_map = {"深圳": '440300', "临沂" : "371300"}
|
||||
|
||||
def load_house_data(city, calc_year_month):
|
||||
end_year_month = int(dateutils.format_date(dateutils.add_months(dateutils.parse_date(str(calc_year_month), "%Y%m"), months=1), "%Y%m"))
|
||||
half_on_year = dateutils.add_months(dateutils.parse_date(str(calc_year_month), "%Y%m"), months=-6)
|
||||
half_on_year_year_month = int(dateutils.format_date(half_on_year, "%Y%m"))
|
||||
sql = f"""
|
||||
select gujia_comm_id as comm_id, count(*) as half_on_year from base.bigdata_forsale where urbn_id = '{city_code_map[city]}'
|
||||
and data_crt_tm >= to_date('{half_on_year_year_month}', 'YYYYMM') and data_crt_tm < to_date('{end_year_month}', 'YYYYMM')
|
||||
group by gujia_comm_id ;
|
||||
"""
|
||||
return pd.read_sql(sql, con=engine236)
|
||||
# ## 按照城市加载楼盘评级数据宽表
|
||||
|
||||
def load_data_by_city(city):
|
||||
def load_data_by_city(city, calc_year_month):
|
||||
comms = load_comms(city=city)
|
||||
stat = load_comm_stat(city=city)
|
||||
stat = load_comm_stat(city, calc_year_month)
|
||||
comms_summary = load_comms_summary(city=city)
|
||||
house_stat = load_house_data(city, calc_year_month)
|
||||
data = comms.merge(stat, how="left", on=['city', 'region', 'name'])
|
||||
data = data.merge(comms_summary, how="left", on=['comm_id'])
|
||||
num_fields = ["supermarket_num", "park_num", "catering_num", "mall_num", "kindergarten_num",
|
||||
@@ -482,10 +492,12 @@ def load_data_by_city(city):
|
||||
|
||||
import multiprocessing
|
||||
|
||||
def process_city(city):
|
||||
def process_city(args):
|
||||
city = args['city']
|
||||
calc_year_month = args['calc_year_month']
|
||||
try:
|
||||
logger.info(f"开始计算楼盘评级:{city}")
|
||||
data = load_data_by_city(city)
|
||||
logger.info(f"开始计算楼盘评级:{city} 年月:{calc_year_month}")
|
||||
data = load_data_by_city(city, calc_year_month)
|
||||
data = comms_rating(data)
|
||||
db_fields = ["comm_id", "city", "region", "name", 'block', 'block_rating', "address", "sqmetre_rent", "yoy",
|
||||
"rent_to_price_ratio", "avg_price", "total_score", "community_quality_score",
|
||||
@@ -546,7 +558,8 @@ def process_city(city):
|
||||
for field in df_simple.columns:
|
||||
if "_rank" in field or "_num" in field:
|
||||
dtype[field] = int
|
||||
updater.bulk_insert_or_update_by_id(df_simple, id_fields=['comm_id'], dtype=dtype)
|
||||
df_simple['date'] = calc_year_month
|
||||
updater.bulk_insert_or_update_by_id(df_simple, id_fields=['comm_id', 'date'], dtype=dtype)
|
||||
logger.info(f"楼盘评级计算结束:{city}")
|
||||
except Exception as e:
|
||||
logging.error(f"{city}楼盘评级计算失败;{e}")
|
||||
@@ -555,12 +568,16 @@ def process_city(city):
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--city', type=str, default='all')
|
||||
parser.add_argument('--month', type=str, default='')
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.city == 'all':
|
||||
cities = areautils.get_all_cities()
|
||||
else:
|
||||
cities = [city.strip() for city in args.city.split(',') if city.strip()]
|
||||
|
||||
if args.month == '':
|
||||
calc_year_month = int(dateutils.format_date(dateutils.add_months(datetime.now(), months=-1), "%Y%m"))
|
||||
else:
|
||||
calc_year_month = int(args.month)
|
||||
args_list = [{"calc_year_month": calc_year_month, "city": city} for city in cities]
|
||||
with multiprocessing.Pool(processes=min(8, len(cities))) as pool:
|
||||
pool.map(process_city, cities)
|
||||
pool.map(process_city, args_list)
|
||||
@@ -12,6 +12,6 @@ with DAG(
|
||||
) as dag:
|
||||
t1 = BashOperator(
|
||||
task_id='community_rating',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/stat/community/community_rate.py --city "{{ dag_run.conf.get("city", "all")}}"'
|
||||
bash_command='python /opt/airflow/dags/repo/dags/stat/community/community_rate.py --city "{{ dag_run.conf.get("city", "all")}}" --month "{{ dag_run.conf.get("month", "")}}"'
|
||||
)
|
||||
t1
|
||||
Reference in New Issue
Block a user