From 7b1792854513b545b958801c9f2752c2d4a0d356 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Thu, 3 Apr 2025 13:56:56 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E8=AF=84=E7=BA=A7=E6=80=BB?= =?UTF-8?q?=E5=88=86100?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../business/community/consumption.py | 14 ++- dags/shop/rent/price.py | 112 ++++++++++++++++++ 2 files changed, 125 insertions(+), 1 deletion(-) create mode 100644 dags/shop/rent/price.py diff --git a/dags/business/business/community/consumption.py b/dags/business/business/community/consumption.py index 416efdb..fb476a4 100644 --- a/dags/business/business/community/consumption.py +++ b/dags/business/business/community/consumption.py @@ -127,7 +127,7 @@ def calc_consumption_rating(community_summary_df): count = 0 # 总评级 - total_rate = 100 + total_rate = 6 if len(df) < total_rate: # 对于样本低于5的直接排名 df[target] = df['consumption_potential'].rank(method='dense', ascending=False).astype(int) @@ -144,11 +144,23 @@ def calc_consumption_rating(community_summary_df): df = gmm_train(total_rate, df, "estate_charge_score", "estate_charge_rating") df = gmm_train(total_rate, df, "relative_price_ranking_score", "relative_price_rating") df = gmm_train(total_rate, df, "avg_cost_score", "avg_cost_rating") + + # 总分数 + total_score = 100 + if len(df) < total_score: + # 对于样本低于5的直接排名 + df["consumption_score"] = df['consumption_potential'].rank(method='dense', ascending=False).astype(int) + else: + # 极值设置 + df = filter_min_max(df) + df = gmm_train(total_rate, df, source, "consumption_score") + # 批量更新 for index, item in df.iterrows(): row = dict() _id = str(item["_id"]) row['consumption_rating'] = item['consumption_rating'] + row['consumption_score'] = item['consumption_score'] row['consumption_potential'] = item['consumption_potential'] row['m_date'] = dateutils.iso_format() row['sub_factor'] = { diff --git a/dags/shop/rent/price.py b/dags/shop/rent/price.py new file mode 100644 index 0000000..13d648c --- /dev/null +++ b/dags/shop/rent/price.py @@ -0,0 +1,112 @@ +import sys + +from goojia_utils import ElasticSearchClient, MAIN_MONGO_CLIENT, dateutils, areautils +import numpy as np +from pymongo import UpdateOne +from datetime import datetime +import logging +import pandas as pd + +log = logging.getLogger(__name__) + +es_client = ElasticSearchClient(host='192.168.0.206') +shop_rent_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'shop_rent_stat') + + +def calc(city, year_month): + if len(str(year_month)) != 6: + raise Exception('年月格式错误yyyyMM') + shop = es_client.scan_by_sql_to_data_frame( + "select * from latest_seawater where cat='shop_rent' and city='" + city + "' and p_date>=" + str( + year_month) + "01 and p_date<=" + str(year_month) + "31 ") + if len(shop) > 0: + shop['day_avg_price'] = shop['day_rent_price'] + # 计算总价的Z-score + shop['Z_Score'] = (shop['day_avg_price'] - shop['day_avg_price'].mean()) / shop['day_avg_price'].std() + # 剔除Z-score超过0.1的异常值 + shop_cleaned = shop[np.abs(shop['Z_Score']) < 0.1] + # 至少样本数大于等于10 + if len(shop_cleaned['day_avg_price']) >= 10: + city_price = round(shop_cleaned['day_avg_price'].median(), 2) + region_median_count_prices = shop_cleaned.groupby(shop_cleaned['region'])['day_avg_price'].agg(['median', 'count']).to_dict() + else: + city_price = None + region_median_count_prices = {'median': {}, 'count': {}} + else: + city_price = None + region_median_count_prices = {'median': {}, 'count': {}} + + actions = list() + + last_month = int( + dateutils.format_date(dateutils.add_months(dateutils.parse_date(str(year_month), '%Y%m'), months=-1), '%Y%m')) + if pd.isna(city_price): + # 处理均价为空情况,用上月均价替代 + stat = shop_rent_stat_conn.find_one({'city': city, 'cat': 'city', 'date': last_month}) + if pd.isna(stat): + return False + last_month_city_price = stat['avg_price'] + if pd.isna(last_month_city_price): + return False + city_price = last_month_city_price + + # 写入MONGODB + action = UpdateOne({'city': city, 'cat': 'city', 'date': year_month}, + {'$set': {'city': city, 'cat': 'city', 'date': year_month, 'avg_price': city_price, + 'updated_date': datetime.now()}}, + upsert=True) + + actions.append(action) + for region in areautils.get_all_districts(city): + region_avg_prices = region_median_count_prices['median'] + region_count = region_median_count_prices['count'] + if region in region_avg_prices and region_count[region] >= 10: + avg_price = region_avg_prices[region] + else: + # 处理均价为空情况,用上月均价替代 + district_stat = shop_rent_stat_conn.find_one({'city': city, 'cat': 'district', 'date': last_month, 'district': region}) + if pd.isna(district_stat): + continue + last_month_region_price = district_stat['avg_price'] + if pd.isna(last_month_region_price): + continue + avg_price = last_month_region_price + + action = UpdateOne({'city': city, 'cat': 'district', 'date': year_month, 'district': region}, + {'$set': {'city': city, 'cat': 'district', 'date': year_month, 'district': region, + 'avg_price': round(avg_price, 2), 'updated_date': datetime.now()}}, + upsert=True) + actions.append(action) + shop_rent_stat_conn.bulk_write(actions) + log.info(f"{city} {year_month} 均价计算完毕!") + + +yearmonth = sys.argv[1] if len(sys.argv) > 1 else '' + +if yearmonth == '': + date = dateutils.add_months(datetime.today(), months=-1) + yearmonth = int(dateutils.format_date(date, '%Y%m')) +try: + for city in areautils.get_all_cities(): + calc(city, yearmonth) +except Exception as e: + log.error(e) + +# # 初始化从2024年9月 +# 创建从2024年9月到2024年12月的日期范围 +# date_range = pd.date_range(start='2024-09-01', end='2025-01-31', freq='M') +# +# # 创建一个包含这些日期的DataFrame +# df = pd.DataFrame({'date': date_range, 'value': range(1, len(date_range) + 1)}) +# +# # 将日期设置为索引 +# df.set_index('date', inplace=True) +# +# # 遍历DataFrame +# for city in areautils.get_all_cities(): +# for date, row in df.iterrows(): +# try: +# date_format = int(dateutils.format_date(date, '%Y%m')) +# calc(city, date_format) +# except Exception as e: +# log.error(e)