import sys from goojia_utils import ElasticSearchClient, MAIN_MONGO_CLIENT, dateutils, areautils import numpy as np from pymongo import UpdateOne from datetime import datetime import logging import pandas as pd log = logging.getLogger(__name__) es_client = ElasticSearchClient(host='192.168.0.206') office_rent_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office_rent_stat') def calc(city, year_month): if len(str(year_month)) != 6: raise Exception('年月格式错误yyyyMM') office = es_client.scan_by_sql_to_data_frame( "select * from latest_seawater where cat='office_rent' and city='" + city + "' and p_date>=" + str( year_month) + "01 and p_date<=" + str(year_month) + "31 ") if len(office) > 0: office['day_avg_price'] = office['avg_price'] / 30 # 计算总价的Z-score office['Z_Score'] = (office['day_avg_price'] - office['day_avg_price'].mean()) / office['day_avg_price'].std() # 剔除Z-score超过0.1的异常值 office_cleaned = office[np.abs(office['Z_Score']) < 0.1] # 至少样本数大于等于10 if len(office_cleaned['day_avg_price']) >= 10: city_price = round(office_cleaned['day_avg_price'].median(), 2) region_median_count_prices = office_cleaned.groupby(office_cleaned['region'])['day_avg_price'].agg(['median', 'count']).to_dict() else: city_price = None region_median_count_prices = {'median': {}, 'count': {}} else: city_price = None region_median_count_prices = {'median': {}, 'count': {}} actions = list() last_month = int( dateutils.format_date(dateutils.add_months(dateutils.parse_date(str(year_month), '%Y%m'), months=-1), '%Y%m')) if pd.isna(city_price): # 处理均价为空情况,用上月均价替代 stat = office_rent_stat_conn.find_one({'city': city, 'cat': 'city', 'date': last_month}) if pd.isna(stat): return False last_month_city_price = stat['avg_price'] if pd.isna(last_month_city_price): return False city_price = last_month_city_price # 写入MONGODB action = UpdateOne({'city': city, 'cat': 'city', 'date': year_month}, {'$set': {'city': city, 'cat': 'city', 'date': year_month, 'avg_price': city_price, 'updated_date': datetime.now()}}, upsert=True) actions.append(action) for region in areautils.get_all_districts(city): region_avg_prices = region_median_count_prices['median'] region_count = region_median_count_prices['count'] if region in region_avg_prices and region_count[region] >= 10: avg_price = region_avg_prices[region] else: # 处理均价为空情况,用上月均价替代 district_stat = office_rent_stat_conn.find_one({'city': city, 'cat': 'district', 'date': last_month, 'district': region}) if pd.isna(district_stat): continue last_month_region_price = district_stat['avg_price'] if pd.isna(last_month_region_price): continue avg_price = last_month_region_price action = UpdateOne({'city': city, 'cat': 'district', 'date': year_month, 'district': region}, {'$set': {'city': city, 'cat': 'district', 'date': year_month, 'district': region, 'avg_price': round(avg_price, 2), 'updated_date': datetime.now()}}, upsert=True) actions.append(action) office_rent_stat_conn.bulk_write(actions) log.info(f"{city} {year_month} 均价计算完毕!") yearmonth = sys.argv[1] if len(sys.argv) > 1 else '' if yearmonth == '': date = dateutils.add_months(datetime.today(), months=-1) yearmonth = int(dateutils.format_date(date, '%Y%m')) try: for city in areautils.get_all_cities(): calc(city, yearmonth) except Exception as e: log.error(e) # # 初始化从2024年9月 # 创建从2024年9月到2024年12月的日期范围 # date_range = pd.date_range(start='2024-09-01', end='2025-01-31', freq='M') # # # 创建一个包含这些日期的DataFrame # df = pd.DataFrame({'date': date_range, 'value': range(1, len(date_range) + 1)}) # # # 将日期设置为索引 # df.set_index('date', inplace=True) # # # 遍历DataFrame # for city in areautils.get_all_cities(): # for date, row in df.iterrows(): # try: # date_format = int(dateutils.format_date(date, '%Y%m')) # calc(city, date_format) # except Exception as e: # log.error(e)