fix(office): 将 fill_base_price 数据库连接移至函数内部以避免 DagBag 导入超时
This commit is contained in:
@@ -1,20 +1,20 @@
|
||||
from goojia_utils import DIDATA_DEV_ELASTIC_CLIENT, dateutils, MAIN_MONGO_CLIENT, threads, areautils
|
||||
import pandas as pd
|
||||
from datetime import datetime
|
||||
from pymongo import UpdateOne
|
||||
import logging
|
||||
import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
office_sell_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office_sell_stat')
|
||||
office_rent_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office_rent_stat')
|
||||
|
||||
|
||||
def fill_base_price(**kwargs):
|
||||
"""
|
||||
写字楼基准价
|
||||
"""
|
||||
from goojia_utils import dateutils, MAIN_MONGO_CLIENT
|
||||
|
||||
office_sell_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office_sell_stat')
|
||||
office_rent_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office_rent_stat')
|
||||
|
||||
# 从 Airflow 的 dag_run.conf 获取参数
|
||||
conf = kwargs.get('dag_run').conf if kwargs.get('dag_run') else {}
|
||||
city = conf.get('city', '')
|
||||
@@ -38,12 +38,16 @@ def fill_base_price(**kwargs):
|
||||
if len(office_rent) <= 0:
|
||||
return
|
||||
if year_month != '':
|
||||
calc_by_yearmonth(office_sell, office_rent, year_month)
|
||||
calc_by_yearmonth(office_sell, office_rent, year_month, office_sell_stat_conn)
|
||||
else:
|
||||
# 对于单个重算的不进行补充,已经自己补充过了
|
||||
pass
|
||||
|
||||
def calc_by_yearmonth(office_sell, office_rent, year_month):
|
||||
def calc_by_yearmonth(office_sell, office_rent, year_month, office_sell_stat_conn=None):
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, threads
|
||||
|
||||
if office_sell_stat_conn is None:
|
||||
office_sell_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'office_sell_stat')
|
||||
office_sell_df = office_sell[['city', 'district', 'name', 'avg_price']]
|
||||
office_sell_df = office_sell_df.rename(columns={'avg_price': 'sell_price'})
|
||||
office_rent_df = office_rent[['city', 'district', 'name', 'avg_price', 'estate_type2', 'bed_kind', 'location']]
|
||||
|
||||
Reference in New Issue
Block a user