diff --git a/dags/stats/evaluate/district_count_thirdavg_deal.py b/dags/stats/evaluate/district_count_thirdavg_deal.py index 35b4cad..59c4254 100644 --- a/dags/stats/evaluate/district_count_thirdavg_deal.py +++ b/dags/stats/evaluate/district_count_thirdavg_deal.py @@ -10,34 +10,8 @@ import sys log = logging.getLogger(__name__) -city_num = sys.argv[1] if len(sys.argv) > 1 else None -if city_num is not None: - log.info("输出参数" + city_num) - -df = pd.DataFrame() -if city_num is not None and int(city_num) == 92: - df = pd.read_excel('/opt/airflow/dags/repo/dags/stat/evaluate/data/估值92城市数据统计.xlsx') -else: - df = pd.read_excel('/opt/airflow/dags/repo/dags/stat/evaluate/data/估值数据统计.xlsx') -year_month_day = dateutils.format_date(datetime.datetime.now()) -path = "/home/airflow/data/evaluate" -aim_file_name = path + '/估值数据统计结果' + year_month_day + '.xlsx' -if not os.path.exists(path): - os.makedirs(path) - -seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') -es206_client = ElasticSearchClient(host='192.168.0.206') -district_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_stat') # 成交 -average_statistics_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'average_statistics') # 第三方均价 -deal_price_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'deal_price') - -seaweed_df = pd.DataFrame( - list(seaweed_conn.find({'cat': 'district', 'status': 0}, {'city': 1, 'region': 1, 'name': 1, 'mapping_alias': 1}) - .batch_size(10000))) -seaweed_df = seaweed_df.rename(columns={'mapping_alias': 'alias'}) - -cities = df[['城市']].drop_duplicates(keep='first')['城市'].values.tolist() +# ================== 核心统计函数(在全局定义,供 main 调用) ================== def count_stat(): """ @@ -128,9 +102,50 @@ def deal_stat(): new_df = pd.merge(new_df, seaweed_half_year_rename_df, how='left', on=['城市']) new_df.to_excel(aim_file_name, index=False) +# ================== 🛡️ 安全运行入口 ================== -log.info("开始统计小区数量、近半年第三方均价、近半年成交信息") -count_stat() -# third_avg_stat() -deal_stat() -log.info("完成统计小区数量、近半年第三方均价、近半年成交信息") +if __name__ == "__main__": + # 1. 只有以脚本形式运行时,才解析命令行参数 + city_num = sys.argv[1] if len(sys.argv) > 1 else None + if city_num is not None: + log.info("输出参数" + city_num) + + # 2. 读取 Excel 配置 + df = pd.DataFrame() + # 增加额外的 try-except 容错,防止意外传入非数字字符串(比如在别处不小心执行时) + try: + if city_num is not None and int(city_num) == 92: + df = pd.read_excel('/opt/airflow/dags/repo/dags/stat/evaluate/data/估值92城市数据统计.xlsx') + else: + df = pd.read_excel('/opt/airflow/dags/repo/dags/stat/evaluate/data/估值数据统计.xlsx') + except ValueError: + log.warning(f"无法将参数 '{city_num}' 转换为整数,将使用默认数据源进行统计。") + df = pd.read_excel('/opt/airflow/dags/repo/dags/stat/evaluate/data/估值数据统计.xlsx') + + year_month_day = dateutils.format_date(datetime.datetime.now()) + path = "/home/airflow/data/evaluate" + aim_file_name = path + '/估值数据统计结果' + year_month_day + '.xlsx' + if not os.path.exists(path): + os.makedirs(path) + + # 3. 延迟初始化数据库连接(避免在 Airflow 解析阶段浪费连接资源) + seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') + es206_client = ElasticSearchClient(host='192.168.0.206') + district_stat_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_stat', 'district_stat') # 成交 + average_statistics_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'average_statistics') # 第三方均价 + deal_price_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'deal_price') + + # 4. 加载 MongoDB 数据 + seaweed_df = pd.DataFrame( + list(seaweed_conn.find({'cat': 'district', 'status': 0}, {'city': 1, 'region': 1, 'name': 1, 'mapping_alias': 1}) + .batch_size(10000))) + seaweed_df = seaweed_df.rename(columns={'mapping_alias': 'alias'}) + + cities = df[['城市']].drop_duplicates(keep='first')['城市'].values.tolist() + + # 5. 执行主统计逻辑 + log.info("开始统计小区数量、近半年第三方均价、近半年成交信息") + count_stat() + # third_avg_stat() + deal_stat() + log.info("完成统计小区数量、近半年第三方均价、近半年成交信息") \ No newline at end of file diff --git a/dags/stats/evaluate/district_sell_property.py b/dags/stats/evaluate/district_sell_property.py index f28477a..9c591f6 100644 --- a/dags/stats/evaluate/district_sell_property.py +++ b/dags/stats/evaluate/district_sell_property.py @@ -8,22 +8,9 @@ import datetime import sys log = logging.getLogger(__name__) -sell_num = sys.argv[1] if len(sys.argv) > 1 else "2" -sell_num = int(sell_num) -log.info("输出参数" + str(sell_num)) -year_month_day = dateutils.format_date(datetime.datetime.now()) -aim_file_name = '/home/airflow/data/evaluate/估值数据统计结果' + year_month_day + '.xlsx' -start_time = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), days=-93)) -one_year_time = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), days=-366)) - -seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') -es206_client = ElasticSearchClient(host='192.168.0.206') - -seaweed_df = pd.DataFrame(list(seaweed_conn.find({'cat': 'district', 'status': 0}, {'city': 1, 'region': 1, 'name': 1, 'mapping_alias': 1}) - .batch_size(10000))) -seaweed_df = seaweed_df.rename(columns={'mapping_alias': 'alias'}) +# ================== 核心函数定义(供 main 调用) ================== def get_sell(df, daysBefore=365): cities = df[['城市']].drop_duplicates(keep='first')['城市'].values.tolist() @@ -115,6 +102,7 @@ def merge_aim(): df = pd.merge(df, need_df, how='left', on=['城市']) df.to_excel(aim_file_name, index=False) + def up_to_nextcloud(): """ 上传统计信息至nextcloud @@ -127,11 +115,40 @@ def up_to_nextcloud(): nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD, json_output=True) nxc.upload_file(aim_file_name, aim_file_path) -log.info("开始统计统计近三个月、近一年样本量及小区属性") -sell_stat() -property_stat() -merge_aim() -log.info("完成统计统计近三个月、近一年样本量及小区属性") -log.info("开始上传至云盘") -up_to_nextcloud() -log.info("云盘上传完成") + +# ================== 🛡️ 安全运行入口 ================== + +if __name__ == "__main__": + # 1. 只有脚本运行时,才解析命令行参数并转为整数 + sell_num_arg = sys.argv[1] if len(sys.argv) > 1 else "2" + try: + sell_num = int(sell_num_arg) + except ValueError: + log.warning(f"无法将参数 '{sell_num_arg}' 转换为整数,将使用默认值 2。") + sell_num = 2 + log.info("输出参数" + str(sell_num)) + + # 2. 统计时间范围及存储文件名定义 + year_month_day = dateutils.format_date(datetime.datetime.now()) + aim_file_name = '/home/airflow/data/evaluate/估值数据统计结果' + year_month_day + '.xlsx' + start_time = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), days=-93)) + one_year_time = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), days=-366)) + + # 3. 延迟数据库初始化(避开 Airflow 解析器扫描) + seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') + es206_client = ElasticSearchClient(host='192.168.0.206') + + # 4. 加载基础数据 + seaweed_df = pd.DataFrame(list(seaweed_conn.find({'cat': 'district', 'status': 0}, {'city': 1, 'region': 1, 'name': 1, 'mapping_alias': 1}) + .batch_size(10000))) + seaweed_df = seaweed_df.rename(columns={'mapping_alias': 'alias'}) + + # 5. 开始核心统计与上传逻辑 + log.info("开始统计统计近三个月、近一年样本量及小区属性") + sell_stat() + property_stat() + merge_aim() + log.info("完成统计统计近三个月、近一年样本量及小区属性") + log.info("开始上传至云盘") + up_to_nextcloud() + log.info("云盘上传完成") \ No newline at end of file