import sys from goojia_utils import ElasticSearchClient, MAIN_MONGO_CLIENT, dateutils, areautils, threads import numpy as np from pymongo import UpdateOne from datetime import datetime import logging import pandas as pd log = logging.getLogger(__name__) es_client = ElasticSearchClient(host='192.168.0.206') office_sell_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office_sell_stat') office_rent_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office_rent_stat') def calc(city, year_month): print(f"开始处理{city}, {year_month}") year_month = int(year_month) query = {'date': year_month, 'cat': 'office'} if pd.notna(city): query['city'] = city office_sell = pd.DataFrame(list(office_sell_stat_conn.find(query))) office_rent = pd.DataFrame(list(office_rent_stat_conn.find(query))) if len(office_rent) <= 0: return office_sell_df = office_sell[['city', 'district', 'name', 'avg_price']] office_sell_df = office_sell_df.rename(columns={'avg_price': 'sell_price'}) office_rent_df = office_rent[['city', 'district', 'name', 'avg_price', 'estate_type2', 'bed_kind', 'location']] office_rent_df = office_rent_df.rename(columns={'avg_price': 'rent_price'}) # 每月动态计算系数,因为不同月份会有差异 merge_df = pd.merge(office_sell_df, office_rent_df, how='left', on=['city', 'district', 'name']) merge_df = merge_df[(merge_df['rent_price'].notna()) & (merge_df['sell_price'].notna())] merge_df['ratio'] = merge_df['sell_price'] / (merge_df['rent_price'] * 30 * 12) merge_df['district_ratio'] = merge_df.groupby(by=['city', 'district'])['ratio'].transform('median') merge_df['city_ratio'] = merge_df.groupby(by=['city'])['ratio'].transform('median') district_radio = dict(zip(merge_df["city"] + "_" + merge_df["district"], merge_df["district_ratio"])) city_radio = dict(zip(merge_df["city"], merge_df["city_ratio"])) # 取没有售价的城市与区域 office_sell_city = pd.DataFrame(list(office_sell_stat_conn.find({'date': year_month, 'cat': 'city'}))) office_sell_city = office_sell_city.rename(columns={'avg_price': 'sell_price'}) office_rent_city = pd.DataFrame(list(office_rent_stat_conn.find({'date': year_month, 'cat': 'city'}))) office_rent_city = office_rent_city.rename(columns={'avg_price': 'rent_price'}) merge2_df = pd.merge(office_rent_city, office_sell_city, how='left', on=['city']) merge2_df = merge2_df[(merge2_df['rent_price'].notna()) & (merge2_df['sell_price'].isna())] def deal_city(item): """ 填充存在租金,不存在售价的数据 """ if item['city'] in city_radio.keys(): item['sell_price'] = int(city_radio[item['city']] * item['rent_price'] * 30 * 12) return item merge2_df = threads.process_dataframe_multithread(merge2_df, deal_city, 10) merge2_df = merge2_df[merge2_df['sell_price'].notna()] year = year_month // 100 month = year_month % 100 prev_date = (year - 1) * 100 + 12 if month == 1 else year_month - 1 ly_date = year_month - 100 prev_city_map = {d.get('city'): d.get('avg_price') for d in office_sell_stat_conn.find({'date': prev_date, 'cat': 'city'}) if d.get('avg_price')} ly_city_map = {d.get('city'): d.get('avg_price') for d in office_sell_stat_conn.find({'date': ly_date, 'cat': 'city'}) if d.get('avg_price')} actions = list() # 写入MONGODB for index, row in merge2_df.iterrows(): sell_p = int(row['sell_price']) prev_p = prev_city_map.get(row['city']) ly_p = ly_city_map.get(row['city']) mom = round((sell_p - prev_p) / prev_p * 100, 2) if prev_p else None yoy = round((sell_p - ly_p) / ly_p * 100, 2) if ly_p else None action = UpdateOne({'city': row['city'], 'cat': 'city', 'date': year_month}, {'$set': {'city': row['city'], 'cat': 'city', 'date': year_month, 'calc_type': 2, 'avg_price': sell_p, 'mom': mom, 'yoy': yoy, 'updated_date': datetime.now()}}, upsert=True) actions.append(action) if len(actions) > 0: office_sell_stat_conn.bulk_write(actions) actions.clear() office_sell_district = pd.DataFrame(list(office_sell_stat_conn.find({'date': year_month, 'cat': 'district'}))) office_sell_district = office_sell_district.rename(columns={'avg_price': 'sell_price'}) office_rent_district = pd.DataFrame(list(office_rent_stat_conn.find({'date': year_month, 'cat': 'district'}))) office_rent_district = office_rent_district.rename(columns={'avg_price': 'rent_price'}) merge2_df = pd.merge(office_rent_district, office_sell_district, how='left', on=['city', 'district']) merge2_df = merge2_df[(merge2_df['rent_price'].notna()) & (merge2_df['sell_price'].isna())] def deal_district(item): """ 填充存在租金,不存在售价的数据 """ if item['city'] + "_" + item['district'] in district_radio.keys(): item['sell_price'] = int(district_radio[item['city'] + "_" + item['district']] * item['rent_price'] * 30 * 12) return item merge2_df = threads.process_dataframe_multithread(merge2_df, deal_district, 10) merge2_df = merge2_df[merge2_df['sell_price'].notna()] prev_dist_map = {(d.get('city'), d.get('district')): d.get('avg_price') for d in office_sell_stat_conn.find({'date': prev_date, 'cat': 'district'}) if d.get('avg_price')} ly_dist_map = {(d.get('city'), d.get('district')): d.get('avg_price') for d in office_sell_stat_conn.find({'date': ly_date, 'cat': 'district'}) if d.get('avg_price')} # 写入MONGODB for index, row in merge2_df.iterrows(): sell_p = int(row['sell_price']) prev_p = prev_dist_map.get((row['city'], row['district'])) ly_p = ly_dist_map.get((row['city'], row['district'])) mom = round((sell_p - prev_p) / prev_p * 100, 2) if prev_p else None yoy = round((sell_p - ly_p) / ly_p * 100, 2) if ly_p else None action = UpdateOne({'city': row['city'], 'district': row['district'], 'cat': 'district', 'date': year_month}, {'$set': {'city': row['city'], 'district': row['district'], 'cat': 'district', 'date': year_month, 'calc_type': 2, 'avg_price': sell_p, 'mom': mom, 'yoy': yoy, 'updated_date': datetime.now()}}, upsert=True) actions.append(action) if len(actions) > 0: office_sell_stat_conn.bulk_write(actions) log.info(f"均价补充完毕!") if __name__ == "__main__": yearmonth = sys.argv[1] if len(sys.argv) > 1 else '' if yearmonth == '': date = dateutils.add_months(datetime.today(), months=-1) yearmonth = int(dateutils.format_date(date, '%Y%m')) try: if type(yearmonth) is str: yearmonth = int(yearmonth) calc(None, yearmonth) except Exception as e: log.error(e) # # 初始化从2024年9月 # 创建从2024年9月到2024年12月的日期范围 # date_range = pd.date_range(start='2025-01-31', end='2025-01-31', freq='M') # # # 创建一个包含这些日期的DataFrame # df = pd.DataFrame({'date': date_range, 'value': range(1, len(date_range) + 1)}) # # # 将日期设置为索引 # df.set_index('date', inplace=True) # # # 遍历DataFrame # for date, row in df.iterrows(): # try: # date_format = int(dateutils.format_date(date, '%Y%m')) # calc(None, date_format) # except Exception as e: # log.error(e)