import pandas as pd from datetime import datetime from pymongo import UpdateOne import logging log = logging.getLogger(__name__) def fill_base_price(**kwargs): """ 写字楼基准价 """ from goojia_utils import dateutils, MAIN_MONGO_CLIENT office_sell_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office_sell_stat') office_rent_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office_rent_stat') # 从 Airflow 的 dag_run.conf 获取参数 conf = kwargs.get('dag_run').conf if kwargs.get('dag_run') else {} city = conf.get('city', '') year_month = conf.get('yearmonth', '') district = conf.get('district', '') office_name = conf.get('name', '') if year_month == "": year_month = dateutils.format_date(dateutils.add_months(datetime.today(), months=-1), "%Y%m") query = {'cat': 'office'} if year_month != '': query['date'] = year_month if city != '': query['city'] = city if district != '' and office_name != '': query['district'] = district query['name'] = office_name office_sell = pd.DataFrame(list(office_sell_stat_conn.find(query))) office_rent = pd.DataFrame(list(office_rent_stat_conn.find(query))) if len(office_rent) <= 0: return if year_month != '': calc_by_yearmonth(office_sell, office_rent, year_month, office_sell_stat_conn) else: # 对于单个重算的不进行补充,已经自己补充过了 pass def calc_by_yearmonth(office_sell, office_rent, year_month, office_sell_stat_conn=None): from goojia_utils import MAIN_MONGO_CLIENT, threads if office_sell_stat_conn is None: office_sell_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office_sell_stat') office_sell_df = office_sell[['city', 'district', 'name', 'avg_price']] office_sell_df = office_sell_df.rename(columns={'avg_price': 'sell_price'}) office_rent_df = office_rent[['city', 'district', 'name', 'avg_price', 'estate_type2', 'bed_kind', 'location']] office_rent_df = office_rent_df.rename(columns={'avg_price': 'rent_price'}) # 每月动态计算系数,因为不同月份会有差异 merge_df = pd.merge(office_sell_df, office_rent_df, how='left', on=['city', 'district', 'name']) merge_df = merge_df[(merge_df['rent_price'].notna()) & (merge_df['sell_price'].notna())] merge_df['ratio'] = merge_df['sell_price'] / (merge_df['rent_price'] * 30 * 12) merge_df['district_ratio'] = merge_df.groupby(by=['city', 'district'])['ratio'].transform('median') merge_df['city_ratio'] = merge_df.groupby(by=['city'])['ratio'].transform('median') district_radio = dict(zip(merge_df["city"] + "_" + merge_df["district"], merge_df["district_ratio"])) city_radio = dict(zip(merge_df["city"], merge_df["city_ratio"])) merge2_df = pd.merge(office_rent_df, office_sell_df, how='left', on=['city', 'district', 'name']) merge2_df = merge2_df[(merge2_df['rent_price'].notna()) & (merge2_df['sell_price'].isna())] def deal(item): """ 填充存在租金,不存在售价的数据 """ if item['city'] + "_" + item['district'] in district_radio.keys(): item['sell_price'] = round( district_radio[item['city'] + "_" + item['district']] * item['rent_price'] * 30 * 12, 0) elif item['city'] in city_radio.keys(): item['sell_price'] = round(city_radio[item['city']] * item['rent_price'] * 30 * 12, 0) return item if len(merge2_df) == 0: return merge2_df = threads.process_dataframe_multithread(merge2_df, deal, 10) merge2_df = merge2_df[merge2_df['sell_price'].notna()] curr_date = int(year_month) year = curr_date // 100 month = curr_date % 100 prev_date = (year - 1) * 100 + 12 if month == 1 else curr_date - 1 ly_date = curr_date - 100 proj = {'_id': 0, 'city': 1, 'district': 1, 'name': 1, 'avg_price': 1} prev_map = {(d.get('city'), d.get('district'), d.get('name')): d.get('avg_price') for d in office_sell_stat_conn.find({'cat': 'office', 'date': prev_date}, proj) if d.get('avg_price')} ly_map = {(d.get('city'), d.get('district'), d.get('name')): d.get('avg_price') for d in office_sell_stat_conn.find({'cat': 'office', 'date': ly_date}, proj) if d.get('avg_price')} actions = list() count = 0 for index, row in merge2_df.iterrows(): curr_city = row['city'] region = row['district'] name = row['name'] sell_price = int(row['sell_price']) estate_type2 = row['estate_type2'] bed_kind = row['bed_kind'] location = row['location'] prev_p = prev_map.get((curr_city, region, name)) ly_p = ly_map.get((curr_city, region, name)) mom = round((sell_price - prev_p) / prev_p * 100, 2) if prev_p else None yoy = round((sell_price - ly_p) / ly_p * 100, 2) if ly_p else None action = UpdateOne({'cat': 'office', 'city': curr_city, 'district': region, 'name': name, 'date': curr_date}, {'$set': {'city': curr_city, 'cat': 'office', 'date': curr_date, 'district': region, 'name': name, 'avg_price': sell_price, 'mom': mom, 'yoy': yoy, 'estate_type2': estate_type2, 'bed_kind': bed_kind, 'location': location, 'calc_type': 5, 'updated_date': datetime.now()}}, upsert=True) actions.append(action) if len(actions) == 5000: office_sell_stat_conn.bulk_write(actions) count += len(actions) print("已处理{}条数据".format(count)) actions.clear() if len(actions) > 0: office_sell_stat_conn.bulk_write(actions) count += len(actions) print("已处理{}条数据".format(count)) print("处理完毕") if __name__ == "__main__": # 测试脚本:直接运行时模拟 Airflow 的上下文 fill_base_price(dag_run=type('DagRun', (), {'conf': {'city': '上海', 'district': '静安', 'name': '凯迪克大厦'}}))