diff --git a/dags/company/company/fill_company_dev.py b/dags/company/company/fill_company_dev.py index ff8ad08..ddd5ea1 100644 --- a/dags/company/company/fill_company_dev.py +++ b/dags/company/company/fill_company_dev.py @@ -4,13 +4,15 @@ from elasticsearch import helpers import numpy as np import re import logging -log = logging.getLogger(__name__) +# 配置日志 +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +logger = logging.getLogger(__name__) """ 通过开发区围栏更新企业开发区 TODO """ def sync(province): - log.info(f"开始同步{province}企业数据的开发区数据") + logger.info(f"开始同步{province}企业数据的开发区数据") # 先临时禁用索引刷新 DIDATA_DEV_ELASTIC_CLIENT.elastic_client.indices.put_settings( index="didata_company", @@ -20,12 +22,13 @@ def sync(province): shape_df = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame(f"select name, shape from development_zone where province='{province}'") actions = list() count = 0 - for index, item in shape_df.iterrows(): - if pd.isna(item['shape']): - continue + for index, item in shape_df[shape_df['shape'].notna()].iterrows(): + logger.info(f"开始查找{item['name']}下企业数据") ret = DIDATA_DEV_ELASTIC_CLIENT.fetch_all_geo_points('didata_company', item['shape']['coordinates'][0], field='location') + com_count = 0 for info in ret: + com_count += 1 doc_info = {'dev': item['name']} action = { "_op_type": "update", @@ -36,18 +39,18 @@ def sync(province): } actions.append(action) - if len(actions) == 10000: - for success, info in helpers.parallel_bulk( - DIDATA_DEV_ELASTIC_CLIENT.elastic_client, - actions, - thread_count=5, # 并行线程数 - chunk_size=1000 # 批量大小 - ): - if not success: - print(f"索引失败: {info}") - count += len(actions) - print("更新:" + str(count)) - actions.clear() + logger.info(f"已查找到该围栏下{com_count}个企业数据") + for success, info in helpers.parallel_bulk( + DIDATA_DEV_ELASTIC_CLIENT.elastic_client, + actions, + thread_count=5, # 并行线程数 + chunk_size=1000 # 批量大小 + ): + if not success: + print(f"索引失败: {info}") + count += len(actions) + print("更新:" + str(count)) + actions.clear() if len(actions) > 0: for success, info in helpers.parallel_bulk( DIDATA_DEV_ELASTIC_CLIENT.elastic_client, @@ -60,7 +63,7 @@ def sync(province): count += len(actions) print("更新:" + str(count)) actions.clear() - log.info(f"完成同步{province}企业数据的开发区数据") + logger.info(f"完成同步{province}企业数据的开发区数据") # 恢复索引刷新 DIDATA_DEV_ELASTIC_CLIENT.elastic_client.indices.put_settings( index="didata_company",