From 1b030c9522ea62705ddb575f0cb492e4fec4a7d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=96=87=E8=89=B3?= Date: Tue, 28 Oct 2025 11:00:02 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=97=A5=E6=9C=9F=E7=B1=BB?= =?UTF-8?q?=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/office/dag_office_rent.py | 14 +++++++--- dags/office/dag_office_rent_base_price.py | 23 ----------------- dags/office/dag_office_sell.py | 28 ++++++++++++++------ dags/office/dag_office_sell_base_price.py | 31 ----------------------- dags/office/rent/base_price.py | 2 +- dags/office/sell/base_price.py | 2 +- dags/office/sell/fill_base_price.py | 2 +- dags/office/sell/fill_price.py | 1 + dags/requirements.txt | 3 ++- 9 files changed, 37 insertions(+), 69 deletions(-) delete mode 100644 dags/office/dag_office_rent_base_price.py delete mode 100644 dags/office/dag_office_sell_base_price.py diff --git a/dags/office/dag_office_rent.py b/dags/office/dag_office_rent.py index 1087ffe..3597407 100644 --- a/dags/office/dag_office_rent.py +++ b/dags/office/dag_office_rent.py @@ -2,6 +2,8 @@ 小区周边配套DAG """ from airflow.operators.bash import BashOperator +from airflow.operators.python import PythonOperator +from office.rent.base_price import calc import pendulum @@ -13,11 +15,17 @@ with DAG( schedule_interval="0 3 26 * *", start_date=pendulum.datetime(2025, 3, 26, tz="Asia/Shanghai"), catchup=True, - description="写字楼租金城市区域均价价格计算" + description="写字楼租金价格计算", + tags=["office", "rent"], ) as dag: t1 = BashOperator( - task_id='office_rent', + task_id='office_city_region_rent_calc', bash_command='python /opt/airflow/dags/repo/dags/office/rent/price.py {{dag_run.conf["yearmonth"] if "yearmonth" in dag_run.conf}} ', ) - t1 + t2 = PythonOperator( + task_id='office_base_rent_calc', + python_callable=calc, + provide_context=True + ) + t1 >> t2 diff --git a/dags/office/dag_office_rent_base_price.py b/dags/office/dag_office_rent_base_price.py deleted file mode 100644 index c9d68cc..0000000 --- a/dags/office/dag_office_rent_base_price.py +++ /dev/null @@ -1,23 +0,0 @@ -""" -写字楼基准价计算 -""" -from airflow.operators.python import PythonOperator - -import pendulum -from office.rent.base_price import calc - -from airflow import DAG - -with DAG( - dag_id="office_rent_base_price_dag", - schedule_interval="0 1 26 * *", - start_date=pendulum.datetime(2025, 2, 28, tz="Asia/Shanghai"), - catchup=True, - description="写字楼租金基准价计算" -) as dag: - t1 = PythonOperator( - task_id='base_price', - python_callable=calc, - provide_context=True - ) - t1 diff --git a/dags/office/dag_office_sell.py b/dags/office/dag_office_sell.py index 8cbbe79..62c5a8f 100644 --- a/dags/office/dag_office_sell.py +++ b/dags/office/dag_office_sell.py @@ -1,5 +1,7 @@ from airflow.operators.bash import BashOperator - +from airflow.operators.python import PythonOperator +from office.sell.base_price import calc +from office.sell.fill_base_price import fill_base_price import pendulum from airflow import DAG @@ -10,16 +12,26 @@ with DAG( schedule_interval="0 3 27 * *", start_date=pendulum.datetime(2025, 3, 26, tz="Asia/Shanghai"), catchup=True, - description="写字楼出售城市区域均价价格计算" + description="写字楼出售价格计算", + tags=["office", "sell"], ) as dag: - t1 = BashOperator( - task_id='office_sell', + t1 = PythonOperator( + task_id='office_sell_base_price_calc', + python_callable=calc, + provide_context=True + ) + t2 = PythonOperator( + task_id='office_no_sample_sell_base_price_calc', + python_callable=fill_base_price, + provide_context=True + ) + t3 = BashOperator( + task_id='office_city_region_sell_base_price_calc', bash_command='python /opt/airflow/dags/repo/dags/office/sell/price.py {{dag_run.conf["yearmonth"] if "yearmonth" in dag_run.conf}} ', ) - - t2 = BashOperator( - task_id='fill_price', + t4 = BashOperator( + task_id='office_city_region_sell_no_sample_base_price_calc', bash_command='python /opt/airflow/dags/repo/dags/office/sell/fill_price.py {{dag_run.conf["yearmonth"] if "yearmonth" in dag_run.conf}}', ) - t1 >> t2 + t1 >> t2 >> t3 >> t4 diff --git a/dags/office/dag_office_sell_base_price.py b/dags/office/dag_office_sell_base_price.py deleted file mode 100644 index 8030ad1..0000000 --- a/dags/office/dag_office_sell_base_price.py +++ /dev/null @@ -1,31 +0,0 @@ -""" -写字楼基准价计算 -""" -from airflow.operators.bash import BashOperator -from airflow.operators.python import PythonOperator -from office.sell.base_price import calc -from office.sell.fill_base_price import fill_base_price - -import pendulum - -from airflow import DAG - -with DAG( - dag_id="office_sell_base_price_dag", - schedule_interval="0 1 27 * *", - start_date=pendulum.datetime(2025, 3, 24, tz="Asia/Shanghai"), - catchup=True, - description="写字楼出售基准价计算" -) as dag: - t1 = PythonOperator( - task_id='calc', - python_callable=calc, - provide_context=True - ) - t2 = PythonOperator( - task_id='fill_base_price', - python_callable=fill_base_price, - provide_context=True - ) - t1 >> t2 - diff --git a/dags/office/rent/base_price.py b/dags/office/rent/base_price.py index 356dcec..8250ce8 100644 --- a/dags/office/rent/base_price.py +++ b/dags/office/rent/base_price.py @@ -31,7 +31,7 @@ def calc(**kwargs): # 从 Airflow 的 dag_run.conf 获取参数 conf = kwargs.get('dag_run').conf if kwargs.get('dag_run') else {} city = conf.get('city', None) - year_month = conf.get('year_month', None) + year_month = conf.get('yearmonth', None) district = conf.get('district', None) office_name = conf.get('name', None) if year_month is None: diff --git a/dags/office/sell/base_price.py b/dags/office/sell/base_price.py index 643014e..8323025 100644 --- a/dags/office/sell/base_price.py +++ b/dags/office/sell/base_price.py @@ -31,7 +31,7 @@ def calc(**kwargs): # 从 Airflow 的 dag_run.conf 获取参数 conf = kwargs.get('dag_run').conf if kwargs.get('dag_run') else {} city = conf.get('city', None) - year_month = conf.get('year_month', None) + year_month = conf.get('yearmonth', None) district = conf.get('district', None) office_name = conf.get('name', None) if year_month is None: diff --git a/dags/office/sell/fill_base_price.py b/dags/office/sell/fill_base_price.py index 7e80f00..5c18d39 100644 --- a/dags/office/sell/fill_base_price.py +++ b/dags/office/sell/fill_base_price.py @@ -18,7 +18,7 @@ def fill_base_price(**kwargs): # 从 Airflow 的 dag_run.conf 获取参数 conf = kwargs.get('dag_run').conf if kwargs.get('dag_run') else {} city = conf.get('city', '') - year_month = conf.get('year_month', '') + year_month = conf.get('yearmonth', '') district = conf.get('district', '') office_name = conf.get('name', '') if year_month == "": diff --git a/dags/office/sell/fill_price.py b/dags/office/sell/fill_price.py index e6484ed..b6550a4 100644 --- a/dags/office/sell/fill_price.py +++ b/dags/office/sell/fill_price.py @@ -15,6 +15,7 @@ office_rent_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office def calc(city, year_month): print(f"开始处理{city}, {year_month}") + year_month = int(year_month) query = {'date': year_month, 'cat': 'office'} if pd.notna(city): query['city'] = city diff --git a/dags/requirements.txt b/dags/requirements.txt index afa4ada..2e8b235 100644 --- a/dags/requirements.txt +++ b/dags/requirements.txt @@ -5,4 +5,5 @@ matplotlib kafka-python==2.0.2 tqdm tabulate==0.9.0 -kafka==1.3.5 \ No newline at end of file +kafka==1.3.5 +apache-airflow==2.5.3 \ No newline at end of file