优化日期类型
This commit is contained in:
@@ -2,6 +2,8 @@
|
||||
小区周边配套DAG
|
||||
"""
|
||||
from airflow.operators.bash import BashOperator
|
||||
from airflow.operators.python import PythonOperator
|
||||
from office.rent.base_price import calc
|
||||
|
||||
import pendulum
|
||||
|
||||
@@ -13,11 +15,17 @@ with DAG(
|
||||
schedule_interval="0 3 26 * *",
|
||||
start_date=pendulum.datetime(2025, 3, 26, tz="Asia/Shanghai"),
|
||||
catchup=True,
|
||||
description="写字楼租金城市区域均价价格计算"
|
||||
description="写字楼租金价格计算",
|
||||
tags=["office", "rent"],
|
||||
) as dag:
|
||||
t1 = BashOperator(
|
||||
task_id='office_rent',
|
||||
task_id='office_city_region_rent_calc',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/office/rent/price.py {{dag_run.conf["yearmonth"] if "yearmonth" in dag_run.conf}} ',
|
||||
)
|
||||
t1
|
||||
t2 = PythonOperator(
|
||||
task_id='office_base_rent_calc',
|
||||
python_callable=calc,
|
||||
provide_context=True
|
||||
)
|
||||
t1 >> t2
|
||||
|
||||
|
||||
@@ -1,23 +0,0 @@
|
||||
"""
|
||||
写字楼基准价计算
|
||||
"""
|
||||
from airflow.operators.python import PythonOperator
|
||||
|
||||
import pendulum
|
||||
from office.rent.base_price import calc
|
||||
|
||||
from airflow import DAG
|
||||
|
||||
with DAG(
|
||||
dag_id="office_rent_base_price_dag",
|
||||
schedule_interval="0 1 26 * *",
|
||||
start_date=pendulum.datetime(2025, 2, 28, tz="Asia/Shanghai"),
|
||||
catchup=True,
|
||||
description="写字楼租金基准价计算"
|
||||
) as dag:
|
||||
t1 = PythonOperator(
|
||||
task_id='base_price',
|
||||
python_callable=calc,
|
||||
provide_context=True
|
||||
)
|
||||
t1
|
||||
@@ -1,5 +1,7 @@
|
||||
from airflow.operators.bash import BashOperator
|
||||
|
||||
from airflow.operators.python import PythonOperator
|
||||
from office.sell.base_price import calc
|
||||
from office.sell.fill_base_price import fill_base_price
|
||||
import pendulum
|
||||
|
||||
from airflow import DAG
|
||||
@@ -10,16 +12,26 @@ with DAG(
|
||||
schedule_interval="0 3 27 * *",
|
||||
start_date=pendulum.datetime(2025, 3, 26, tz="Asia/Shanghai"),
|
||||
catchup=True,
|
||||
description="写字楼出售城市区域均价价格计算"
|
||||
description="写字楼出售价格计算",
|
||||
tags=["office", "sell"],
|
||||
) as dag:
|
||||
t1 = BashOperator(
|
||||
task_id='office_sell',
|
||||
t1 = PythonOperator(
|
||||
task_id='office_sell_base_price_calc',
|
||||
python_callable=calc,
|
||||
provide_context=True
|
||||
)
|
||||
t2 = PythonOperator(
|
||||
task_id='office_no_sample_sell_base_price_calc',
|
||||
python_callable=fill_base_price,
|
||||
provide_context=True
|
||||
)
|
||||
t3 = BashOperator(
|
||||
task_id='office_city_region_sell_base_price_calc',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/office/sell/price.py {{dag_run.conf["yearmonth"] if "yearmonth" in dag_run.conf}} ',
|
||||
)
|
||||
|
||||
t2 = BashOperator(
|
||||
task_id='fill_price',
|
||||
t4 = BashOperator(
|
||||
task_id='office_city_region_sell_no_sample_base_price_calc',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/office/sell/fill_price.py {{dag_run.conf["yearmonth"] if "yearmonth" in dag_run.conf}}',
|
||||
)
|
||||
t1 >> t2
|
||||
t1 >> t2 >> t3 >> t4
|
||||
|
||||
|
||||
@@ -1,31 +0,0 @@
|
||||
"""
|
||||
写字楼基准价计算
|
||||
"""
|
||||
from airflow.operators.bash import BashOperator
|
||||
from airflow.operators.python import PythonOperator
|
||||
from office.sell.base_price import calc
|
||||
from office.sell.fill_base_price import fill_base_price
|
||||
|
||||
import pendulum
|
||||
|
||||
from airflow import DAG
|
||||
|
||||
with DAG(
|
||||
dag_id="office_sell_base_price_dag",
|
||||
schedule_interval="0 1 27 * *",
|
||||
start_date=pendulum.datetime(2025, 3, 24, tz="Asia/Shanghai"),
|
||||
catchup=True,
|
||||
description="写字楼出售基准价计算"
|
||||
) as dag:
|
||||
t1 = PythonOperator(
|
||||
task_id='calc',
|
||||
python_callable=calc,
|
||||
provide_context=True
|
||||
)
|
||||
t2 = PythonOperator(
|
||||
task_id='fill_base_price',
|
||||
python_callable=fill_base_price,
|
||||
provide_context=True
|
||||
)
|
||||
t1 >> t2
|
||||
|
||||
@@ -31,7 +31,7 @@ def calc(**kwargs):
|
||||
# 从 Airflow 的 dag_run.conf 获取参数
|
||||
conf = kwargs.get('dag_run').conf if kwargs.get('dag_run') else {}
|
||||
city = conf.get('city', None)
|
||||
year_month = conf.get('year_month', None)
|
||||
year_month = conf.get('yearmonth', None)
|
||||
district = conf.get('district', None)
|
||||
office_name = conf.get('name', None)
|
||||
if year_month is None:
|
||||
|
||||
@@ -31,7 +31,7 @@ def calc(**kwargs):
|
||||
# 从 Airflow 的 dag_run.conf 获取参数
|
||||
conf = kwargs.get('dag_run').conf if kwargs.get('dag_run') else {}
|
||||
city = conf.get('city', None)
|
||||
year_month = conf.get('year_month', None)
|
||||
year_month = conf.get('yearmonth', None)
|
||||
district = conf.get('district', None)
|
||||
office_name = conf.get('name', None)
|
||||
if year_month is None:
|
||||
|
||||
@@ -18,7 +18,7 @@ def fill_base_price(**kwargs):
|
||||
# 从 Airflow 的 dag_run.conf 获取参数
|
||||
conf = kwargs.get('dag_run').conf if kwargs.get('dag_run') else {}
|
||||
city = conf.get('city', '')
|
||||
year_month = conf.get('year_month', '')
|
||||
year_month = conf.get('yearmonth', '')
|
||||
district = conf.get('district', '')
|
||||
office_name = conf.get('name', '')
|
||||
if year_month == "":
|
||||
|
||||
@@ -15,6 +15,7 @@ office_rent_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office
|
||||
|
||||
def calc(city, year_month):
|
||||
print(f"开始处理{city}, {year_month}")
|
||||
year_month = int(year_month)
|
||||
query = {'date': year_month, 'cat': 'office'}
|
||||
if pd.notna(city):
|
||||
query['city'] = city
|
||||
|
||||
@@ -6,3 +6,4 @@ kafka-python==2.0.2
|
||||
tqdm
|
||||
tabulate==0.9.0
|
||||
kafka==1.3.5
|
||||
apache-airflow==2.5.3
|
||||
Reference in New Issue
Block a user