From 8eb6cb3876b4d55ed9bf0f889755c9d7685a3566 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 26 Mar 2025 13:51:41 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=9F=8E=E5=B8=82=E5=8C=BA?= =?UTF-8?q?=E5=9F=9F=E5=9D=87=E4=BB=B7=E8=A1=A5=E5=85=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/office/sell/base_price.py | 10 +-- dags/office/sell/fill_price.py | 132 +++++++++++++++++++++++++++++++++ 2 files changed, 137 insertions(+), 5 deletions(-) create mode 100644 dags/office/sell/fill_price.py diff --git a/dags/office/sell/base_price.py b/dags/office/sell/base_price.py index c41e6a4..e38f0ed 100644 --- a/dags/office/sell/base_price.py +++ b/dags/office/sell/base_price.py @@ -80,7 +80,7 @@ def calc(city, year_month): city = item['city'] region = item['region'] name = item['name'] - avg_price = round(item['day_avg_price'], 0) + avg_price = int(item['day_avg_price']) location = { "type": "Point", "coordinates": [item['lng'] / math.pow(10, 10), item['lat'] / math.pow(10, 10)] @@ -144,7 +144,7 @@ def calc(city, year_month): # 计算平均值 average_price = np.mean(avg_prices) if len(avg_prices) > 0 else 0 if average_price > 0: - avg_price = round(average_price, 0) + avg_price = int(average_price) action = UpdateOne({'cat': 'office', 'city': city, 'district': region, 'name': name, 'date': year_month}, {'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region, 'name': name, @@ -209,7 +209,7 @@ def calc(city, year_month): sum_price += new_price count += 1 if count > 0: - avg_price = round(sum_price / count, 0) + avg_price = int(sum_price / count) action = UpdateOne({'cat': 'office', 'city': city, 'district': region, 'name': name, 'date': year_month}, {'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region, 'name': name, @@ -248,7 +248,7 @@ def calc(city, year_month): sum_price += new_price count += 1 if count > 0: - avg_price = round(sum_price / count, 0) + avg_price = int(sum_price / count) action = UpdateOne({'cat': 'office', 'city': city, 'district': region, 'name': name, 'date': year_month}, {'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region, 'name': name, @@ -320,7 +320,7 @@ def calc(city, year_month): # 计算平均值 average_price = np.mean(avg_prices) if len(avg_prices) > 0 else 0 if average_price > 0: - avg_price = round(average_price, 0) + avg_price = int(average_price) action = UpdateOne({'cat': 'office', 'city': city, 'district': region, 'name': name, 'date': year_month}, {'$set': {'city': city, 'cat': 'office', 'date': year_month, 'district': region, 'name': name, diff --git a/dags/office/sell/fill_price.py b/dags/office/sell/fill_price.py new file mode 100644 index 0000000..cf4480f --- /dev/null +++ b/dags/office/sell/fill_price.py @@ -0,0 +1,132 @@ +import sys + +from goojia_utils import ElasticSearchClient, MAIN_MONGO_CLIENT, dateutils, areautils, threads +import numpy as np +from pymongo import UpdateOne +from datetime import datetime +import logging +import pandas as pd + +log = logging.getLogger(__name__) + +es_client = ElasticSearchClient(host='192.168.0.206') +office_sell_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office_sell_stat') +office_rent_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office_rent_stat') + + +def calc(city, year_month): + query = {'date': year_month, 'cat': 'office'} + if pd.notna(city): + query['city'] = city + office_sell = pd.DataFrame(list(office_sell_stat_conn.find(query))) + office_rent = pd.DataFrame(list(office_rent_stat_conn.find(query))) + + if len(office_rent) <= 0: + return + office_sell_df = office_sell[['city', 'district', 'name', 'avg_price']] + office_sell_df = office_sell_df.rename(columns={'avg_price': 'sell_price'}) + office_rent_df = office_rent[['city', 'district', 'name', 'avg_price', 'estate_type2', 'bed_kind', 'location']] + office_rent_df = office_rent_df.rename(columns={'avg_price': 'rent_price'}) + # 每月动态计算系数,因为不同月份会有差异 + merge_df = pd.merge(office_sell_df, office_rent_df, how='left', on=['city', 'district', 'name']) + merge_df = merge_df[(merge_df['rent_price'].notna()) & (merge_df['sell_price'].notna())] + + merge_df['ratio'] = merge_df['sell_price'] / (merge_df['rent_price'] * 30 * 12) + merge_df['district_ratio'] = merge_df.groupby(by=['city', 'district'])['ratio'].transform('median') + merge_df['city_ratio'] = merge_df.groupby(by=['city'])['ratio'].transform('median') + + district_radio = dict(zip(merge_df["city"] + "_" + merge_df["district"], merge_df["district_ratio"])) + city_radio = dict(zip(merge_df["city"], merge_df["city_ratio"])) + + # 取没有售价的城市与区域 + office_sell_city = pd.DataFrame(list(office_sell_stat_conn.find({'date': year_month, 'cat': 'city'}))) + office_sell_city = office_sell_city.rename(columns={'avg_price': 'sell_price'}) + office_rent_city = pd.DataFrame(list(office_rent_stat_conn.find({'date': year_month, 'cat': 'city'}))) + office_rent_city = office_rent_city.rename(columns={'avg_price': 'rent_price'}) + + merge2_df = pd.merge(office_rent_city, office_sell_city, how='left', on=['city']) + merge2_df = merge2_df[(merge2_df['rent_price'].notna()) & (merge2_df['sell_price'].isna())] + + def deal_city(item): + """ + 填充存在租金,不存在售价的数据 + """ + if item['city'] in city_radio.keys(): + item['sell_price'] = int(city_radio[item['city']] * item['rent_price'] * 30 * 12) + return item + + merge2_df = threads.process_dataframe_multithread(merge2_df, deal_city, 10) + merge2_df = merge2_df[merge2_df['sell_price'].notna()] + + actions = list() + # 写入MONGODB + for index, row in merge2_df.iterrows(): + action = UpdateOne({'city': row['city'], 'cat': 'city', 'date': year_month}, + {'$set': {'city': row['city'], 'cat': 'city', 'date': year_month, 'calc_type': 2, 'avg_price': row['sell_price'], + 'updated_date': datetime.now()}}, + upsert=True) + actions.append(action) + if len(actions) > 0: + office_sell_stat_conn.bulk_write(actions) + actions.clear() + + office_sell_district = pd.DataFrame(list(office_sell_stat_conn.find({'date': year_month, 'cat': 'district'}))) + office_sell_district = office_sell_district.rename(columns={'avg_price': 'sell_price'}) + office_rent_district = pd.DataFrame(list(office_rent_stat_conn.find({'date': year_month, 'cat': 'district'}))) + office_rent_district = office_rent_district.rename(columns={'avg_price': 'rent_price'}) + + merge2_df = pd.merge(office_rent_district, office_sell_district, how='left', on=['city', 'district']) + merge2_df = merge2_df[(merge2_df['rent_price'].notna()) & (merge2_df['sell_price'].isna())] + def deal_district(item): + """ + 填充存在租金,不存在售价的数据 + """ + if item['city'] + "_" + item['district'] in district_radio.keys(): + item['sell_price'] = int(district_radio[item['city'] + "_" + item['district']] * item['rent_price'] * 30 * 12) + return item + + merge2_df = threads.process_dataframe_multithread(merge2_df, deal_district, 10) + merge2_df = merge2_df[merge2_df['sell_price'].notna()] + + # 写入MONGODB + for index, row in merge2_df.iterrows(): + action = UpdateOne({'city': row['city'], 'district': row['district'], 'cat': 'district', 'date': year_month}, + {'$set': {'city': row['city'], 'district': row['district'], 'cat': 'district', 'date': year_month, 'calc_type': 2, 'avg_price': row['sell_price'], + 'updated_date': datetime.now()}}, + upsert=True) + actions.append(action) + if len(actions) > 0: + office_sell_stat_conn.bulk_write(actions) + log.info(f"均价补充完毕!") + + +yearmonth = sys.argv[1] if len(sys.argv) > 1 else '' + +if yearmonth == '': + date = dateutils.add_months(datetime.today(), months=-1) + yearmonth = int(dateutils.format_date(date, '%Y%m')) +try: + if type(yearmonth) is str: + yearmonth = int(yearmonth) + calc(None, yearmonth) +except Exception as e: + log.error(e) + +# # 初始化从2024年9月 +# 创建从2024年9月到2024年12月的日期范围 +# date_range = pd.date_range(start='2024-09-01', end='2025-01-31', freq='M') +# +# # 创建一个包含这些日期的DataFrame +# df = pd.DataFrame({'date': date_range, 'value': range(1, len(date_range) + 1)}) +# +# # 将日期设置为索引 +# df.set_index('date', inplace=True) +# +# # 遍历DataFrame +# for city in areautils.get_all_cities(): +# for date, row in df.iterrows(): +# try: +# date_format = int(dateutils.format_date(date, '%Y%m')) +# calc(city, date_format) +# except Exception as e: +# log.error(e)