From 0c751f413a4ea81e6d6470d557c847b5e9155378 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 26 Mar 2025 13:15:44 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=86=99=E5=AD=97=E6=A5=BC?= =?UTF-8?q?=E5=94=AE=E4=BB=B7=E5=A1=AB=E5=85=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/office/dag_office_rent_base_price.py | 2 +- dags/office/dag_office_sell_base_price.py | 8 +- dags/office/sell/fill_base_price.py | 93 +++++++++++++++++++++++ 3 files changed, 100 insertions(+), 3 deletions(-) create mode 100644 dags/office/sell/fill_base_price.py diff --git a/dags/office/dag_office_rent_base_price.py b/dags/office/dag_office_rent_base_price.py index cdb57f5..dfef620 100644 --- a/dags/office/dag_office_rent_base_price.py +++ b/dags/office/dag_office_rent_base_price.py @@ -9,7 +9,7 @@ from airflow import DAG with DAG( dag_id="office_rent_base_price_dag", - schedule_interval="0 1 27 * *", + schedule_interval="0 1 26 * *", start_date=pendulum.datetime(2025, 2, 28, tz="Asia/Shanghai"), catchup=True, description="写字楼基准价计算" diff --git a/dags/office/dag_office_sell_base_price.py b/dags/office/dag_office_sell_base_price.py index 26ede96..81550cb 100644 --- a/dags/office/dag_office_sell_base_price.py +++ b/dags/office/dag_office_sell_base_price.py @@ -9,7 +9,7 @@ from airflow import DAG with DAG( dag_id="office_sell_base_price_dag", - schedule_interval="0 1 26 * *", + schedule_interval="0 1 27 * *", start_date=pendulum.datetime(2025, 3, 24, tz="Asia/Shanghai"), catchup=True, description="写字楼出售基准价计算" @@ -18,5 +18,9 @@ with DAG( task_id='sell_base_price', bash_command='python /opt/airflow/dags/repo/dags/office/sell/base_price.py {{dag_run.conf["yearmonth"] if "yearmonth" in dag_run.conf}}', ) - t1 + t2 = BashOperator( + task_id='fill_base_price', + bash_command='python /opt/airflow/dags/repo/dags/office/sell/fill_base_price.py {{dag_run.conf["yearmonth"] if "yearmonth" in dag_run.conf}}', + ) + t1 >> t2 diff --git a/dags/office/sell/fill_base_price.py b/dags/office/sell/fill_base_price.py new file mode 100644 index 0000000..32b08a2 --- /dev/null +++ b/dags/office/sell/fill_base_price.py @@ -0,0 +1,93 @@ +from goojia_utils import DIDATA_DEV_ELASTIC_CLIENT, dateutils, MAIN_MONGO_CLIENT, threads, areautils +import pandas as pd +from datetime import datetime +from pymongo import UpdateOne +import sys + +office_sell_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office_sell_stat') +office_rent_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office_rent_stat') + +def calc(city, year_month): + query = {'date': year_month, 'cat': 'office'} + if pd.notna(city): + query['city'] = city + office_sell = pd.DataFrame(list(office_sell_stat_conn.find(query))) + office_rent = pd.DataFrame(list(office_rent_stat_conn.find(query))) + + if len(office_rent) <= 0: + return + office_sell_df = office_sell[['city', 'district', 'name', 'avg_price']] + office_sell_df = office_sell_df.rename(columns={'avg_price': 'sell_price'}) + office_rent_df = office_rent[['city', 'district', 'name', 'avg_price', 'estate_type2', 'bed_kind','location']] + office_rent_df = office_rent_df.rename(columns={'avg_price': 'rent_price'}) + # 每月动态计算系数,因为不同月份会有差异 + merge_df = pd.merge(office_sell_df, office_rent_df, how='left', on=['city', 'district', 'name']) + merge_df = merge_df[(merge_df['rent_price'].notna()) & (merge_df['sell_price'].notna())] + + merge_df['ratio'] = merge_df['sell_price'] / (merge_df['rent_price'] * 30 * 12) + merge_df['district_ratio'] = merge_df.groupby(by=['city', 'district'])['ratio'].transform('median') + merge_df['city_ratio'] = merge_df.groupby(by=['city'])['ratio'].transform('median') + + district_radio = dict(zip(merge_df["city"] + "_" + merge_df["district"], merge_df["district_ratio"])) + city_radio = dict(zip(merge_df["city"], merge_df["city_ratio"])) + + merge2_df = pd.merge(office_rent_df, office_sell_df, how='left', on=['city', 'district', 'name']) + merge2_df = merge2_df[(merge2_df['rent_price'].notna()) & (merge2_df['sell_price'].isna())] + + def deal(item): + """ + 填充存在租金,不存在售价的数据 + """ + if item['city'] + "_" + item['district'] in district_radio.keys(): + item['sell_price'] = round(district_radio[item['city'] + "_" + item['district']] * item['rent_price'] * 30 * 12, 0) + elif item['city'] in city_radio.keys(): + item['sell_price'] = round(city_radio[item['city']] * item['rent_price'] * 30 * 12, 0) + return item + + merge2_df = threads.process_dataframe_multithread(merge2_df, deal, 10) + merge2_df = merge2_df[merge2_df['sell_price'].notna()] + + actions = list() + count = 0 + for index, row in merge2_df.iterrows(): + curr_city = row['city'] + region = row['district'] + name = row['name'] + sell_price = row['sell_price'] + estate_type2 = row['estate_type2'] + bed_kind = row['bed_kind'] + location = row['location'] + action = UpdateOne({'cat': 'office', 'city': curr_city, 'district': region, 'name': name, 'date': year_month}, + {'$set': {'city': curr_city, 'cat': 'office', 'date': year_month, 'district': region, + 'name': name, + 'avg_price': sell_price, 'estate_type2': estate_type2, 'bed_kind': bed_kind, + 'location': location, + 'calc_type': 5, 'updated_date': datetime.now()}}, + upsert=True) + actions.append(action) + if len(actions) == 5000: + office_sell_stat_conn.bulk_write(actions) + count += len(actions) + print("已处理{}条数据".format(count)) + actions.clear() + if len(actions) > 0: + office_sell_stat_conn.bulk_write(actions) + count += len(actions) + print("已处理{}条数据".format(count)) + print("处理完毕") + + +yearmonth = sys.argv[1] if len(sys.argv) > 1 else '' + +if yearmonth == '': + date = dateutils.add_months(datetime.today(), months=-1) + yearmonth = int(dateutils.format_date(date, '%Y%m')) +try: + if type(yearmonth) is str: + yearmonth = int(yearmonth) + # for curr_city in areautils.get_all_cities(): + # print("正在处理城市:{}".format(curr_city)) + calc(None, yearmonth) +except Exception as e: + print(e) +