fix(office): 延迟导入业务模块并补充 .airflowignore 及 __main__ 保护以解决 DagBag 导入超时

This commit is contained in:
王文艳
2026-10-19 09:38:44 +08:00
parent 0b7caf6a9f
commit fb8f6400be
6 changed files with 68 additions and 37 deletions
+17 -1
View File
@@ -1,9 +1,25 @@
poi/business/*
stat/evaluate/*
stat/community/*
stats/evaluate/*
stats/community/*
stats/aiprice/*
stats/cityPrice/*
stats/test/*
mt/supply/*
business/business/*
jh/merge/*
jh/para/*
jh/price/*
db/*
jh/sample/*
db/*
office/rent/*
office/sell/*
shop/rent/*
shop/sell/*
company/company/*
ai/stock/*
price_exponents/region_price_month.py
price_exponents/seaweed_price_month.py
test/*
train/*
+6 -2
View File
@@ -3,11 +3,15 @@
"""
from airflow.providers.standard.operators.bash import BashOperator
from airflow.operators.python import PythonOperator, BranchPythonOperator
from office.rent.base_price import calc
import pendulum
from airflow import DAG
def run_base_rent_calc(**kwargs):
from office.rent.base_price import calc
return calc(**kwargs)
def choose_task(**context):
"""根据 dag_run.conf 决定执行哪个任务"""
conf = context['dag_run'].conf or {}
@@ -50,7 +54,7 @@ with DAG(
# 任务2:写字楼租金计算
t2 = PythonOperator(
task_id='office_base_rent_calc',
python_callable=calc,
python_callable=run_base_rent_calc,
# 🛡️ 已移除 provide_context=True,防止 Airflow 3.3.0 报错崩溃
)
branch_task >> [t1, t2]
+12 -4
View File
@@ -1,13 +1,21 @@
from airflow.providers.standard.operators.bash import BashOperator
from airflow.operators.python import PythonOperator, BranchPythonOperator
from airflow.operators.empty import EmptyOperator
from office.sell.base_price import calc
from office.sell.fill_base_price import fill_base_price
import pendulum
from airflow import DAG
def run_base_price_calc(**kwargs):
from office.sell.base_price import calc
return calc(**kwargs)
def run_fill_base_price(**kwargs):
from office.sell.fill_base_price import fill_base_price
return fill_base_price(**kwargs)
def choose_task(**context):
"""根据 dag_run.conf 决定执行哪个任务"""
conf = context['dag_run'].conf or {}
@@ -56,14 +64,14 @@ with DAG(
# 任务1: 写字楼出售基础价格计算
t1 = PythonOperator(
task_id='office_sell_base_price_calc',
python_callable=calc,
python_callable=run_base_price_calc,
# 🛡️ 已安全移除 provide_context=True
)
# 任务2: 无样本写字楼出售基础价格填充
t2 = PythonOperator(
task_id='office_no_sample_sell_base_price_calc',
python_callable=fill_base_price,
python_callable=run_fill_base_price,
# 🛡️ 已安全移除 provide_context=True
)
+10 -9
View File
@@ -81,16 +81,17 @@ def calc(city, year_month):
log.info(f"{city} {year_month} 均价计算完毕!")
yearmonth = sys.argv[1] if len(sys.argv) > 1 else ''
if __name__ == "__main__":
yearmonth = sys.argv[1] if len(sys.argv) > 1 else ''
if yearmonth == '':
date = dateutils.add_months(datetime.today(), months=-1)
yearmonth = int(dateutils.format_date(date, '%Y%m'))
try:
for city in areautils.get_all_cities():
calc(city, yearmonth)
except Exception as e:
log.error(e)
if yearmonth == '':
date = dateutils.add_months(datetime.today(), months=-1)
yearmonth = int(dateutils.format_date(date, '%Y%m'))
try:
for city in areautils.get_all_cities():
calc(city, yearmonth)
except Exception as e:
log.error(e)
# # 初始化从2024年9月
# 创建从2024年9月到2024年12月的日期范围
+11 -10
View File
@@ -101,17 +101,18 @@ def calc(city, year_month):
log.info(f"均价补充完毕!")
yearmonth = sys.argv[1] if len(sys.argv) > 1 else ''
if __name__ == "__main__":
yearmonth = sys.argv[1] if len(sys.argv) > 1 else ''
if yearmonth == '':
date = dateutils.add_months(datetime.today(), months=-1)
yearmonth = int(dateutils.format_date(date, '%Y%m'))
try:
if type(yearmonth) is str:
yearmonth = int(yearmonth)
calc(None, yearmonth)
except Exception as e:
log.error(e)
if yearmonth == '':
date = dateutils.add_months(datetime.today(), months=-1)
yearmonth = int(dateutils.format_date(date, '%Y%m'))
try:
if type(yearmonth) is str:
yearmonth = int(yearmonth)
calc(None, yearmonth)
except Exception as e:
log.error(e)
# # 初始化从2024年9月
# 创建从2024年9月到2024年12月的日期范围
+12 -11
View File
@@ -80,18 +80,19 @@ def calc(city, year_month):
office_sell_stat_conn.bulk_write(actions)
log.info(f"{city} {year_month} 均价计算完毕!")
yearmonth = sys.argv[1] if len(sys.argv) > 1 else ''
if __name__ == "__main__":
yearmonth = sys.argv[1] if len(sys.argv) > 1 else ''
if yearmonth == '':
date = dateutils.add_months(datetime.today(), months=-1)
yearmonth = int(dateutils.format_date(date, '%Y%m'))
try:
if type(yearmonth) is str:
yearmonth = int(yearmonth)
for city in areautils.get_all_cities():
calc(city, yearmonth)
except Exception as e:
log.error(e)
if yearmonth == '':
date = dateutils.add_months(datetime.today(), months=-1)
yearmonth = int(dateutils.format_date(date, '%Y%m'))
try:
if type(yearmonth) is str:
yearmonth = int(yearmonth)
for city in areautils.get_all_cities():
calc(city, yearmonth)
except Exception as e:
log.error(e)
# # 初始化从2024年9月
# 创建从2024年9月到2024年12月的日期范围