From 3975d029f843d92c3809fbca7015bb00ffb202f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 20 Jun 2025 13:26:11 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=BC=81=E4=B8=9A=E5=BC=80?= =?UTF-8?q?=E5=8F=91=E5=8C=BA=E5=90=8C=E6=AD=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/company/company/fill_company_dev.py | 26 ++++++++++++++---------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/dags/company/company/fill_company_dev.py b/dags/company/company/fill_company_dev.py index d131ee3..d8de094 100644 --- a/dags/company/company/fill_company_dev.py +++ b/dags/company/company/fill_company_dev.py @@ -20,6 +20,9 @@ def sync(province): ) shape_df = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame(f"select name, shape from development_zone where province='{province}'") + if len(shape_df) == 0 or 'shape' not in shape_df.columns or len(shape_df[shape_df['shape'].notna()]) == 0: + logger.info(f"{province}无围栏数据") + return actions = list() count = 0 for index, item in shape_df[shape_df['shape'].notna()].iterrows(): @@ -40,17 +43,18 @@ def sync(province): actions.append(action) logger.info(f"已查找到该围栏下{com_count}个企业数据") - for success, info in helpers.parallel_bulk( - DIDATA_DEV_ELASTIC_CLIENT.elastic_client, - actions, - thread_count=5, # 并行线程数 - chunk_size=1000 # 批量大小 - ): - if not success: - print(f"索引失败: {info}") - count += len(actions) - print("更新:" + str(count)) - actions.clear() + if len(actions) > 0: + for success, info in helpers.parallel_bulk( + DIDATA_DEV_ELASTIC_CLIENT.elastic_client, + actions, + thread_count=5, # 并行线程数 + chunk_size=1000 # 批量大小 + ): + if not success: + print(f"索引失败: {info}") + count += len(actions) + print("更新:" + str(count)) + actions.clear() if len(actions) > 0: for success, info in helpers.parallel_bulk( DIDATA_DEV_ELASTIC_CLIENT.elastic_client,