fix(office): 延迟导入业务模块并补充 .airflowignore 及 __main__ 保护以解决 DagBag 导入超时
This commit is contained in:
@@ -1,9 +1,25 @@
|
||||
poi/business/*
|
||||
stat/evaluate/*
|
||||
stat/community/*
|
||||
stats/evaluate/*
|
||||
stats/community/*
|
||||
stats/aiprice/*
|
||||
stats/cityPrice/*
|
||||
stats/test/*
|
||||
mt/supply/*
|
||||
business/business/*
|
||||
jh/merge/*
|
||||
jh/para/*
|
||||
jh/price/*
|
||||
jh/sample/*
|
||||
db/*
|
||||
office/rent/*
|
||||
office/sell/*
|
||||
shop/rent/*
|
||||
shop/sell/*
|
||||
company/company/*
|
||||
ai/stock/*
|
||||
price_exponents/region_price_month.py
|
||||
price_exponents/seaweed_price_month.py
|
||||
test/*
|
||||
train/*
|
||||
@@ -3,11 +3,15 @@
|
||||
"""
|
||||
from airflow.providers.standard.operators.bash import BashOperator
|
||||
from airflow.operators.python import PythonOperator, BranchPythonOperator
|
||||
from office.rent.base_price import calc
|
||||
import pendulum
|
||||
from airflow import DAG
|
||||
|
||||
|
||||
def run_base_rent_calc(**kwargs):
|
||||
from office.rent.base_price import calc
|
||||
return calc(**kwargs)
|
||||
|
||||
|
||||
def choose_task(**context):
|
||||
"""根据 dag_run.conf 决定执行哪个任务"""
|
||||
conf = context['dag_run'].conf or {}
|
||||
@@ -50,7 +54,7 @@ with DAG(
|
||||
# 任务2:写字楼租金计算
|
||||
t2 = PythonOperator(
|
||||
task_id='office_base_rent_calc',
|
||||
python_callable=calc,
|
||||
python_callable=run_base_rent_calc,
|
||||
# 🛡️ 已移除 provide_context=True,防止 Airflow 3.3.0 报错崩溃
|
||||
)
|
||||
branch_task >> [t1, t2]
|
||||
@@ -1,13 +1,21 @@
|
||||
from airflow.providers.standard.operators.bash import BashOperator
|
||||
from airflow.operators.python import PythonOperator, BranchPythonOperator
|
||||
from airflow.operators.empty import EmptyOperator
|
||||
from office.sell.base_price import calc
|
||||
from office.sell.fill_base_price import fill_base_price
|
||||
import pendulum
|
||||
|
||||
from airflow import DAG
|
||||
|
||||
|
||||
def run_base_price_calc(**kwargs):
|
||||
from office.sell.base_price import calc
|
||||
return calc(**kwargs)
|
||||
|
||||
|
||||
def run_fill_base_price(**kwargs):
|
||||
from office.sell.fill_base_price import fill_base_price
|
||||
return fill_base_price(**kwargs)
|
||||
|
||||
|
||||
def choose_task(**context):
|
||||
"""根据 dag_run.conf 决定执行哪个任务"""
|
||||
conf = context['dag_run'].conf or {}
|
||||
@@ -56,14 +64,14 @@ with DAG(
|
||||
# 任务1: 写字楼出售基础价格计算
|
||||
t1 = PythonOperator(
|
||||
task_id='office_sell_base_price_calc',
|
||||
python_callable=calc,
|
||||
python_callable=run_base_price_calc,
|
||||
# 🛡️ 已安全移除 provide_context=True
|
||||
)
|
||||
|
||||
# 任务2: 无样本写字楼出售基础价格填充
|
||||
t2 = PythonOperator(
|
||||
task_id='office_no_sample_sell_base_price_calc',
|
||||
python_callable=fill_base_price,
|
||||
python_callable=run_fill_base_price,
|
||||
# 🛡️ 已安全移除 provide_context=True
|
||||
)
|
||||
|
||||
|
||||
@@ -81,15 +81,16 @@ def calc(city, year_month):
|
||||
log.info(f"{city} {year_month} 均价计算完毕!")
|
||||
|
||||
|
||||
yearmonth = sys.argv[1] if len(sys.argv) > 1 else ''
|
||||
if __name__ == "__main__":
|
||||
yearmonth = sys.argv[1] if len(sys.argv) > 1 else ''
|
||||
|
||||
if yearmonth == '':
|
||||
if yearmonth == '':
|
||||
date = dateutils.add_months(datetime.today(), months=-1)
|
||||
yearmonth = int(dateutils.format_date(date, '%Y%m'))
|
||||
try:
|
||||
try:
|
||||
for city in areautils.get_all_cities():
|
||||
calc(city, yearmonth)
|
||||
except Exception as e:
|
||||
except Exception as e:
|
||||
log.error(e)
|
||||
|
||||
# # 初始化从2024年9月
|
||||
|
||||
@@ -101,16 +101,17 @@ def calc(city, year_month):
|
||||
log.info(f"均价补充完毕!")
|
||||
|
||||
|
||||
yearmonth = sys.argv[1] if len(sys.argv) > 1 else ''
|
||||
if __name__ == "__main__":
|
||||
yearmonth = sys.argv[1] if len(sys.argv) > 1 else ''
|
||||
|
||||
if yearmonth == '':
|
||||
if yearmonth == '':
|
||||
date = dateutils.add_months(datetime.today(), months=-1)
|
||||
yearmonth = int(dateutils.format_date(date, '%Y%m'))
|
||||
try:
|
||||
try:
|
||||
if type(yearmonth) is str:
|
||||
yearmonth = int(yearmonth)
|
||||
calc(None, yearmonth)
|
||||
except Exception as e:
|
||||
except Exception as e:
|
||||
log.error(e)
|
||||
|
||||
# # 初始化从2024年9月
|
||||
|
||||
@@ -80,17 +80,18 @@ def calc(city, year_month):
|
||||
office_sell_stat_conn.bulk_write(actions)
|
||||
log.info(f"{city} {year_month} 均价计算完毕!")
|
||||
|
||||
yearmonth = sys.argv[1] if len(sys.argv) > 1 else ''
|
||||
if __name__ == "__main__":
|
||||
yearmonth = sys.argv[1] if len(sys.argv) > 1 else ''
|
||||
|
||||
if yearmonth == '':
|
||||
if yearmonth == '':
|
||||
date = dateutils.add_months(datetime.today(), months=-1)
|
||||
yearmonth = int(dateutils.format_date(date, '%Y%m'))
|
||||
try:
|
||||
try:
|
||||
if type(yearmonth) is str:
|
||||
yearmonth = int(yearmonth)
|
||||
for city in areautils.get_all_cities():
|
||||
calc(city, yearmonth)
|
||||
except Exception as e:
|
||||
except Exception as e:
|
||||
log.error(e)
|
||||
|
||||
# # 初始化从2024年9月
|
||||
|
||||
Reference in New Issue
Block a user