From 047b826691cd73273212601225edc5f5b984d9c5 Mon Sep 17 00:00:00 2001 From: wangwenyan Date: Thu, 3 Jul 2025 16:03:07 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=A5=BC=E7=9B=98=E8=AF=84?= =?UTF-8?q?=E7=BA=A7=E7=AE=97=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/stat/community/community_rate.py | 81 ++++++++++++++++----------- dags/stat/dag_community_rating.py | 2 +- 2 files changed, 50 insertions(+), 33 deletions(-) diff --git a/dags/stat/community/community_rate.py b/dags/stat/community/community_rate.py index 266ad48..a455e14 100644 --- a/dags/stat/community/community_rate.py +++ b/dags/stat/community/community_rate.py @@ -1,10 +1,10 @@ import argparse import logging +import os import re from datetime import datetime import numpy as np -import math import pandas as pd from goojia_utils import ElasticSearchClient from goojia_utils import MAIN_MONGO_CLIENT, MongoDataFrameUpdater @@ -14,10 +14,16 @@ from sklearn.mixture import GaussianMixture from sklearn.neighbors import BallTree from sklearn.preprocessing import MinMaxScaler from sklearn.preprocessing import StandardScaler +from sqlalchemy import create_engine + +os.environ["NLS_LANG"] = "AMERICAN_AMERICA.AL32UTF8" +engine236 = create_engine("postgresql+psycopg2://postgres:Goojia123456@192.168.0.235:5433/fangjia", echo=False) + + logger = logging.getLogger(__name__) seaweed_client = MAIN_MONGO_CLIENT.get_collection("fangjia", "seaweed") -stat_client = MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "district_stat") +stat_client = MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "district_sell_stat") city_sell_stat_client = MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "city_sell_stat") sqmetre_rent_client = MAIN_MONGO_CLIENT.get_collection("fangjia", "sqmetre_rent") community_rate_client = MAIN_MONGO_CLIENT.get_collection("fangjia_stat", "community_rate") @@ -228,18 +234,17 @@ def load_comms(city): # ## 加载小区价格数据 -def load_comm_stat(city): - calc_year_month = int(dateutils.format_date(datetime.now(), "%Y%m")) - stat = pd.DataFrame(list(stat_client.find({"weekend": dateutils.get_last_weekend(), "city": city}, - {"city": 1, "region": 1, "district_name": 1, "avg_price_2b": 1, - "_id": 0}))) - stat = stat.rename(columns={"district_name": "name", "avg_price_2b": "avg_price"}) +def load_comm_stat(city, calc_year_month): + one_year_ago = dateutils.add_months(dateutils.parse_date(str(calc_year_month),"%Y%m"), years=-1) + half_on_year = dateutils.add_months(dateutils.parse_date(str(calc_year_month),"%Y%m"), months=-6) + one_year_ago_year_month = int(dateutils.format_date(one_year_ago, "%Y%m")) + half_on_year_year_month = int(dateutils.format_date(half_on_year, "%Y%m")) + + stat = pd.DataFrame(list(stat_client.find({"date": calc_year_month, "city": city}, {"city": 1, "region": 1, "district_name": 1, "avg_price": 1, "_id": 0}))) + stat = stat.rename(columns={"district_name": "name"}) stat['org_avg_price'] = stat["avg_price"] - one_year_before_stat = pd.DataFrame(list( - stat_client.find({"weekend": dateutils.add_months(dateutils.get_last_weekend(), days=-52 * 7), "city": city}, - {"city": 1, "region": 1, "district_name": 1, "avg_price_2b": 1, "_id": 0}))) - one_year_before_stat = one_year_before_stat.rename( - columns={"district_name": "name", "avg_price_2b": "one_year_before_avg_price"}) + one_year_before_stat = pd.DataFrame(list(stat_client.find({"date": one_year_ago_year_month, "city": city}, {"city": 1, "region": 1, "district_name": 1, "avg_price": 1, "_id": 0}))) + one_year_before_stat = one_year_before_stat.rename(columns={"district_name": "name", "avg_price": "one_year_before_avg_price"}) stat = stat.merge(one_year_before_stat, on=['city', 'region', 'name'], how="left") stat['yoy'] = (stat['avg_price'] - stat['one_year_before_avg_price']) * 100 / stat['one_year_before_avg_price'] stat['org_yoy'] = stat["yoy"] @@ -248,11 +253,8 @@ def load_comm_stat(city): stat['resistance_coef'] = stat['resistance_coef'] + 2 stat['yoy'] = stat['yoy'].apply(lambda x: round(x, 2)) del stat['one_year_before_avg_price'] - half_on_year_stat = pd.DataFrame(list( - stat_client.find({"weekend": dateutils.add_months(dateutils.get_last_weekend(), days=-26 * 7), "city": city}, - {"city": 1, "region": 1, "district_name": 1, "avg_price_2b": 1, "_id": 0}))) - half_on_year_stat = half_on_year_stat.rename( - columns={"district_name": "name", "avg_price_2b": "half_on_year_avg_price"}) + half_on_year_stat = pd.DataFrame(list(stat_client.find({"date": half_on_year_year_month, "city": city}, {"city": 1, "region": 1, "district_name": 1, "avg_price": 1, "_id": 0}))) + half_on_year_stat = half_on_year_stat.rename(columns={"district_name": "name", "avg_price": "half_on_year_avg_price"}) if half_on_year_stat.empty: return pd.DataFrame() stat = stat.merge(half_on_year_stat, on=['city', 'region', 'name'], how="left") @@ -260,11 +262,7 @@ def load_comm_stat(city): stat['half_on_year'] = stat['yoy'].fillna(0) stat['half_on_year'] = stat['yoy'].apply(lambda x: round(x, 2)) del stat['half_on_year_avg_price'] - - firstDayOfThisMonth = dateutils.parse_date(dateutils.format_date(datetime.utcnow(), "%Y-%m"), "%Y-%m") - lastDayOfPreMonth = dateutils.add_months(firstDayOfThisMonth, days=-1) - sqmetre_rent = pd.DataFrame(list(sqmetre_rent_client.find({"weekend": lastDayOfPreMonth, "cat": "district", "city": city}, - {"city": 1, "region": 1, "district_name": 1, "avg_price": 1, "_id": 0}))) + sqmetre_rent = pd.DataFrame(list(sqmetre_rent_client.find({"s_date": calc_year_month, "cat": "district", "city": city},{"city": 1, "region": 1, "district_name": 1, "avg_price": 1, "_id": 0}))) sqmetre_rent = sqmetre_rent.rename(columns={"district_name": "name", "avg_price": "sqmetre_rent"}) stat = stat.merge(sqmetre_rent, on=['city', 'region', 'name'], how='left') stat['rent_to_price_ratio'] = stat['sqmetre_rent'] * 12 / stat['avg_price'] * 100 @@ -452,13 +450,25 @@ def comms_rating(data): data[f'{score_factor}'] = data[f'{score_factor}'].apply(lambda x: round(x * 100, 6)) return data +city_code_map = {"深圳": '440300', "临沂" : "371300"} +def load_house_data(city, calc_year_month): + end_year_month = int(dateutils.format_date(dateutils.add_months(dateutils.parse_date(str(calc_year_month), "%Y%m"), months=1), "%Y%m")) + half_on_year = dateutils.add_months(dateutils.parse_date(str(calc_year_month), "%Y%m"), months=-6) + half_on_year_year_month = int(dateutils.format_date(half_on_year, "%Y%m")) + sql = f""" + select gujia_comm_id as comm_id, count(*) as half_on_year from base.bigdata_forsale where urbn_id = '{city_code_map[city]}' + and data_crt_tm >= to_date('{half_on_year_year_month}', 'YYYYMM') and data_crt_tm < to_date('{end_year_month}', 'YYYYMM') + group by gujia_comm_id ; + """ + return pd.read_sql(sql, con=engine236) # ## 按照城市加载楼盘评级数据宽表 -def load_data_by_city(city): +def load_data_by_city(city, calc_year_month): comms = load_comms(city=city) - stat = load_comm_stat(city=city) + stat = load_comm_stat(city, calc_year_month) comms_summary = load_comms_summary(city=city) + house_stat = load_house_data(city, calc_year_month) data = comms.merge(stat, how="left", on=['city', 'region', 'name']) data = data.merge(comms_summary, how="left", on=['comm_id']) num_fields = ["supermarket_num", "park_num", "catering_num", "mall_num", "kindergarten_num", @@ -482,10 +492,12 @@ def load_data_by_city(city): import multiprocessing -def process_city(city): +def process_city(args): + city = args['city'] + calc_year_month = args['calc_year_month'] try: - logger.info(f"开始计算楼盘评级:{city}") - data = load_data_by_city(city) + logger.info(f"开始计算楼盘评级:{city} 年月:{calc_year_month}") + data = load_data_by_city(city, calc_year_month) data = comms_rating(data) db_fields = ["comm_id", "city", "region", "name", 'block', 'block_rating', "address", "sqmetre_rent", "yoy", "rent_to_price_ratio", "avg_price", "total_score", "community_quality_score", @@ -546,7 +558,8 @@ def process_city(city): for field in df_simple.columns: if "_rank" in field or "_num" in field: dtype[field] = int - updater.bulk_insert_or_update_by_id(df_simple, id_fields=['comm_id'], dtype=dtype) + df_simple['date'] = calc_year_month + updater.bulk_insert_or_update_by_id(df_simple, id_fields=['comm_id', 'date'], dtype=dtype) logger.info(f"楼盘评级计算结束:{city}") except Exception as e: logging.error(f"{city}楼盘评级计算失败;{e}") @@ -555,12 +568,16 @@ def process_city(city): if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('--city', type=str, default='all') + parser.add_argument('--month', type=str, default='') args = parser.parse_args() - if args.city == 'all': cities = areautils.get_all_cities() else: cities = [city.strip() for city in args.city.split(',') if city.strip()] - + if args.month == '': + calc_year_month = int(dateutils.format_date(dateutils.add_months(datetime.now(), months=-1), "%Y%m")) + else: + calc_year_month = int(args.month) + args_list = [{"calc_year_month": calc_year_month, "city": city} for city in cities] with multiprocessing.Pool(processes=min(8, len(cities))) as pool: - pool.map(process_city, cities) \ No newline at end of file + pool.map(process_city, args_list) \ No newline at end of file diff --git a/dags/stat/dag_community_rating.py b/dags/stat/dag_community_rating.py index a946f53..1186583 100644 --- a/dags/stat/dag_community_rating.py +++ b/dags/stat/dag_community_rating.py @@ -12,6 +12,6 @@ with DAG( ) as dag: t1 = BashOperator( task_id='community_rating', - bash_command='python /opt/airflow/dags/repo/dags/stat/community/community_rate.py --city "{{ dag_run.conf.get("city", "all")}}"' + bash_command='python /opt/airflow/dags/repo/dags/stat/community/community_rate.py --city "{{ dag_run.conf.get("city", "all")}}" --month "{{ dag_run.conf.get("month", "")}}"' ) t1 \ No newline at end of file