优化日期类型

This commit is contained in:
2025-10-28 11:00:02 +08:00
parent 2f6a40d64a
commit 1b030c9522
9 changed files with 37 additions and 69 deletions
+11 -3
View File
@@ -2,6 +2,8 @@
小区周边配套DAG
"""
from airflow.operators.bash import BashOperator
from airflow.operators.python import PythonOperator
from office.rent.base_price import calc
import pendulum
@@ -13,11 +15,17 @@ with DAG(
schedule_interval="0 3 26 * *",
start_date=pendulum.datetime(2025, 3, 26, tz="Asia/Shanghai"),
catchup=True,
description="写字楼租金城市区域均价价格计算"
description="写字楼租金价格计算",
tags=["office", "rent"],
) as dag:
t1 = BashOperator(
task_id='office_rent',
task_id='office_city_region_rent_calc',
bash_command='python /opt/airflow/dags/repo/dags/office/rent/price.py {{dag_run.conf["yearmonth"] if "yearmonth" in dag_run.conf}} ',
)
t1
t2 = PythonOperator(
task_id='office_base_rent_calc',
python_callable=calc,
provide_context=True
)
t1 >> t2
-23
View File
@@ -1,23 +0,0 @@
"""
写字楼基准价计算
"""
from airflow.operators.python import PythonOperator
import pendulum
from office.rent.base_price import calc
from airflow import DAG
with DAG(
dag_id="office_rent_base_price_dag",
schedule_interval="0 1 26 * *",
start_date=pendulum.datetime(2025, 2, 28, tz="Asia/Shanghai"),
catchup=True,
description="写字楼租金基准价计算"
) as dag:
t1 = PythonOperator(
task_id='base_price',
python_callable=calc,
provide_context=True
)
t1
+20 -8
View File
@@ -1,5 +1,7 @@
from airflow.operators.bash import BashOperator
from airflow.operators.python import PythonOperator
from office.sell.base_price import calc
from office.sell.fill_base_price import fill_base_price
import pendulum
from airflow import DAG
@@ -10,16 +12,26 @@ with DAG(
schedule_interval="0 3 27 * *",
start_date=pendulum.datetime(2025, 3, 26, tz="Asia/Shanghai"),
catchup=True,
description="写字楼出售城市区域均价价格计算"
description="写字楼出售价格计算",
tags=["office", "sell"],
) as dag:
t1 = BashOperator(
task_id='office_sell',
t1 = PythonOperator(
task_id='office_sell_base_price_calc',
python_callable=calc,
provide_context=True
)
t2 = PythonOperator(
task_id='office_no_sample_sell_base_price_calc',
python_callable=fill_base_price,
provide_context=True
)
t3 = BashOperator(
task_id='office_city_region_sell_base_price_calc',
bash_command='python /opt/airflow/dags/repo/dags/office/sell/price.py {{dag_run.conf["yearmonth"] if "yearmonth" in dag_run.conf}} ',
)
t2 = BashOperator(
task_id='fill_price',
t4 = BashOperator(
task_id='office_city_region_sell_no_sample_base_price_calc',
bash_command='python /opt/airflow/dags/repo/dags/office/sell/fill_price.py {{dag_run.conf["yearmonth"] if "yearmonth" in dag_run.conf}}',
)
t1 >> t2
t1 >> t2 >> t3 >> t4
-31
View File
@@ -1,31 +0,0 @@
"""
写字楼基准价计算
"""
from airflow.operators.bash import BashOperator
from airflow.operators.python import PythonOperator
from office.sell.base_price import calc
from office.sell.fill_base_price import fill_base_price
import pendulum
from airflow import DAG
with DAG(
dag_id="office_sell_base_price_dag",
schedule_interval="0 1 27 * *",
start_date=pendulum.datetime(2025, 3, 24, tz="Asia/Shanghai"),
catchup=True,
description="写字楼出售基准价计算"
) as dag:
t1 = PythonOperator(
task_id='calc',
python_callable=calc,
provide_context=True
)
t2 = PythonOperator(
task_id='fill_base_price',
python_callable=fill_base_price,
provide_context=True
)
t1 >> t2
+1 -1
View File
@@ -31,7 +31,7 @@ def calc(**kwargs):
# 从 Airflow 的 dag_run.conf 获取参数
conf = kwargs.get('dag_run').conf if kwargs.get('dag_run') else {}
city = conf.get('city', None)
year_month = conf.get('year_month', None)
year_month = conf.get('yearmonth', None)
district = conf.get('district', None)
office_name = conf.get('name', None)
if year_month is None:
+1 -1
View File
@@ -31,7 +31,7 @@ def calc(**kwargs):
# 从 Airflow 的 dag_run.conf 获取参数
conf = kwargs.get('dag_run').conf if kwargs.get('dag_run') else {}
city = conf.get('city', None)
year_month = conf.get('year_month', None)
year_month = conf.get('yearmonth', None)
district = conf.get('district', None)
office_name = conf.get('name', None)
if year_month is None:
+1 -1
View File
@@ -18,7 +18,7 @@ def fill_base_price(**kwargs):
# 从 Airflow 的 dag_run.conf 获取参数
conf = kwargs.get('dag_run').conf if kwargs.get('dag_run') else {}
city = conf.get('city', '')
year_month = conf.get('year_month', '')
year_month = conf.get('yearmonth', '')
district = conf.get('district', '')
office_name = conf.get('name', '')
if year_month == "":
+1
View File
@@ -15,6 +15,7 @@ office_rent_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office
def calc(city, year_month):
print(f"开始处理{city}, {year_month}")
year_month = int(year_month)
query = {'date': year_month, 'cat': 'office'}
if pd.notna(city):
query['city'] = city
+1
View File
@@ -6,3 +6,4 @@ kafka-python==2.0.2
tqdm
tabulate==0.9.0
kafka==1.3.5
apache-airflow==2.5.3