新增城市区域均价补充

This commit is contained in:
沈健
2025-03-26 13:51:41 +08:00
parent 812888eb79
commit b36485473c
+132
View File
@@ -0,0 +1,132 @@
import sys
from goojia_utils import ElasticSearchClient, MAIN_MONGO_CLIENT, dateutils, areautils, threads
import numpy as np
from pymongo import UpdateOne
from datetime import datetime
import logging
import pandas as pd
log = logging.getLogger(__name__)
es_client = ElasticSearchClient(host='192.168.0.206')
office_sell_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office_sell_stat')
office_rent_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office_rent_stat')
def calc(city, year_month):
query = {'date': year_month, 'cat': 'office'}
if pd.notna(city):
query['city'] = city
office_sell = pd.DataFrame(list(office_sell_stat_conn.find(query)))
office_rent = pd.DataFrame(list(office_rent_stat_conn.find(query)))
if len(office_rent) <= 0:
return
office_sell_df = office_sell[['city', 'district', 'name', 'avg_price']]
office_sell_df = office_sell_df.rename(columns={'avg_price': 'sell_price'})
office_rent_df = office_rent[['city', 'district', 'name', 'avg_price', 'estate_type2', 'bed_kind', 'location']]
office_rent_df = office_rent_df.rename(columns={'avg_price': 'rent_price'})
# 每月动态计算系数,因为不同月份会有差异
merge_df = pd.merge(office_sell_df, office_rent_df, how='left', on=['city', 'district', 'name'])
merge_df = merge_df[(merge_df['rent_price'].notna()) & (merge_df['sell_price'].notna())]
merge_df['ratio'] = merge_df['sell_price'] / (merge_df['rent_price'] * 30 * 12)
merge_df['district_ratio'] = merge_df.groupby(by=['city', 'district'])['ratio'].transform('median')
merge_df['city_ratio'] = merge_df.groupby(by=['city'])['ratio'].transform('median')
district_radio = dict(zip(merge_df["city"] + "_" + merge_df["district"], merge_df["district_ratio"]))
city_radio = dict(zip(merge_df["city"], merge_df["city_ratio"]))
# 取没有售价的城市与区域
office_sell_city = pd.DataFrame(list(office_sell_stat_conn.find({'date': year_month, 'cat': 'city'})))
office_sell_city = office_sell_city.rename(columns={'avg_price': 'sell_price'})
office_rent_city = pd.DataFrame(list(office_rent_stat_conn.find({'date': year_month, 'cat': 'city'})))
office_rent_city = office_rent_city.rename(columns={'avg_price': 'rent_price'})
merge2_df = pd.merge(office_rent_city, office_sell_city, how='left', on=['city'])
merge2_df = merge2_df[(merge2_df['rent_price'].notna()) & (merge2_df['sell_price'].isna())]
def deal_city(item):
"""
填充存在租金,不存在售价的数据
"""
if item['city'] in city_radio.keys():
item['sell_price'] = round(city_radio[item['city']] * item['rent_price'] * 30 * 12, 0)
return item
merge2_df = threads.process_dataframe_multithread(merge2_df, deal_city, 10)
merge2_df = merge2_df[merge2_df['sell_price'].notna()]
actions = list()
# 写入MONGODB
for index, row in merge2_df.iterrows():
action = UpdateOne({'city': row['city'], 'cat': 'city', 'date': year_month},
{'$set': {'city': row['city'], 'cat': 'city', 'date': year_month, 'calc_type': 2, 'avg_price': row['sell_price'],
'updated_date': datetime.now()}},
upsert=True)
actions.append(action)
if len(actions) > 0:
office_sell_stat_conn.bulk_write(actions)
actions.clear()
office_sell_district = pd.DataFrame(list(office_sell_stat_conn.find({'date': year_month, 'cat': 'district'})))
office_sell_district = office_sell_district.rename(columns={'avg_price': 'sell_price'})
office_rent_district = pd.DataFrame(list(office_rent_stat_conn.find({'date': year_month, 'cat': 'district'})))
office_rent_district = office_rent_district.rename(columns={'avg_price': 'rent_price'})
merge2_df = pd.merge(office_rent_district, office_sell_district, how='left', on=['city', 'district'])
merge2_df = merge2_df[(merge2_df['rent_price'].notna()) & (merge2_df['sell_price'].isna())]
def deal_district(item):
"""
填充存在租金,不存在售价的数据
"""
if item['city'] + "_" + item['district'] in district_radio.keys():
item['sell_price'] = round(district_radio[item['city'] + "_" + item['district']] * item['rent_price'] * 30 * 12, 0)
return item
merge2_df = threads.process_dataframe_multithread(merge2_df, deal_district, 10)
merge2_df = merge2_df[merge2_df['sell_price'].notna()]
# 写入MONGODB
for index, row in merge2_df.iterrows():
action = UpdateOne({'city': row['city'], 'district': row['district'], 'cat': 'district', 'date': year_month},
{'$set': {'city': row['city'], 'district': row['district'], 'cat': 'city', 'date': year_month, 'calc_type': 2, 'avg_price': row['sell_price'],
'updated_date': datetime.now()}},
upsert=True)
actions.append(action)
if len(actions) > 0:
office_sell_stat_conn.bulk_write(actions)
log.info(f"均价补充完毕!")
yearmonth = sys.argv[1] if len(sys.argv) > 1 else ''
if yearmonth == '':
date = dateutils.add_months(datetime.today(), months=-1)
yearmonth = int(dateutils.format_date(date, '%Y%m'))
try:
if type(yearmonth) is str:
yearmonth = int(yearmonth)
calc(None, yearmonth)
except Exception as e:
log.error(e)
# # 初始化从2024年9月
# 创建从2024年9月到2024年12月的日期范围
# date_range = pd.date_range(start='2024-09-01', end='2025-01-31', freq='M')
#
# # 创建一个包含这些日期的DataFrame
# df = pd.DataFrame({'date': date_range, 'value': range(1, len(date_range) + 1)})
#
# # 将日期设置为索引
# df.set_index('date', inplace=True)
#
# # 遍历DataFrame
# for city in areautils.get_all_cities():
# for date, row in df.iterrows():
# try:
# date_format = int(dateutils.format_date(date, '%Y%m'))
# calc(city, date_format)
# except Exception as e:
# log.error(e)