diff --git a/dags/company/company/fill_company_dev.py b/dags/company/company/fill_company_dev.py index d131ee3..d8de094 100644 --- a/dags/company/company/fill_company_dev.py +++ b/dags/company/company/fill_company_dev.py @@ -20,6 +20,9 @@ def sync(province): ) shape_df = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame(f"select name, shape from development_zone where province='{province}'") + if len(shape_df) == 0 or 'shape' not in shape_df.columns or len(shape_df[shape_df['shape'].notna()]) == 0: + logger.info(f"{province}无围栏数据") + return actions = list() count = 0 for index, item in shape_df[shape_df['shape'].notna()].iterrows(): @@ -40,17 +43,18 @@ def sync(province): actions.append(action) logger.info(f"已查找到该围栏下{com_count}个企业数据") - for success, info in helpers.parallel_bulk( - DIDATA_DEV_ELASTIC_CLIENT.elastic_client, - actions, - thread_count=5, # 并行线程数 - chunk_size=1000 # 批量大小 - ): - if not success: - print(f"索引失败: {info}") - count += len(actions) - print("更新:" + str(count)) - actions.clear() + if len(actions) > 0: + for success, info in helpers.parallel_bulk( + DIDATA_DEV_ELASTIC_CLIENT.elastic_client, + actions, + thread_count=5, # 并行线程数 + chunk_size=1000 # 批量大小 + ): + if not success: + print(f"索引失败: {info}") + count += len(actions) + print("更新:" + str(count)) + actions.clear() if len(actions) > 0: for success, info in helpers.parallel_bulk( DIDATA_DEV_ELASTIC_CLIENT.elastic_client,