diff --git a/dags/company/company/fill_company_comm.py b/dags/company/company/fill_company_comm.py index e5a04b4..3d447b2 100644 --- a/dags/company/company/fill_company_comm.py +++ b/dags/company/company/fill_company_comm.py @@ -5,21 +5,25 @@ from goojia_utils import DIDATA_DEV_ELASTIC_CLIENT, areautils import pandas as pd from elasticsearch import helpers import logging -log = logging.getLogger(__name__) +# 配置日志 +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +logger = logging.getLogger(__name__) def sync(province): - log.info(f"开始同步{province}企业数据的居委村委数据") + logger.info(f"开始同步{province}企业数据的居委村委数据") shape_df = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame( f"select city_name, area_name, street_name, name, shape from community where province_name='{province}'") if len(shape_df) == 0 or 'shape' not in shape_df.columns or len(shape_df[shape_df['shape'].notna()]) == 0: - log.info(f"{province}无围栏数据") + logger.info(f"{province}无围栏数据") return + logger.info(f"已加载{len(shape_df)}条围栏") # 先临时禁用索引刷新 DIDATA_DEV_ELASTIC_CLIENT.elastic_client.indices.put_settings( index="didata_company", body={"index": {"refresh_interval": "-1"}} ) - ids = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame(f"select company_name, committee from didata_company where province='{province}'") + #ids = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame(f"select company_name, committee from didata_company where province='{province}'") + ids = pd.DataFrame({}) if len(ids) > 0: company_ids = ids['_id'].to_list() else: @@ -28,8 +32,11 @@ def sync(province): count = 0 errors = list() for index, item in shape_df[shape_df['shape'].notna()].iterrows(): + logger.info(f"开始查找{item['city_name'], item['area_name'], item['name']}下企业数据") ret = DIDATA_DEV_ELASTIC_CLIENT.fetch_all_geo_points('didata_enterprise', item['shape']['coordinates'][0], field='gaode_location') + com_count = 0 for info in ret: + com_count += 1 source = info['_source'] if 'district' in source and (source['district'] != item['area_name']): errors.append(source) @@ -58,20 +65,20 @@ def sync(province): } actions.append(action) - if len(actions) == 10000: - for success, info in helpers.parallel_bulk( - DIDATA_DEV_ELASTIC_CLIENT.elastic_client, - actions, - thread_count=5, # 并行线程数 - refresh=False, - chunk_size=1000 # 批量大小 - ): - if not success: - print(f"索引失败: {info}") - count += len(actions) - print("更新:" + str(count)) - actions.clear() #pd.DataFrame(errors)[['registered_address', 'company_name', 'province', 'city', 'district', 'gaode_location']].to_csv(f'{province}经纬度错误地址.csv', index=False) + logger.info(f"已查找到该围栏下{com_count}个企业数据") + for success, info in helpers.parallel_bulk( + DIDATA_DEV_ELASTIC_CLIENT.elastic_client, + actions, + thread_count=5, # 并行线程数 + refresh=False, + chunk_size=1000 # 批量大小 + ): + if not success: + logger.info(f"索引失败: {info}") + count += len(actions) + logger.info("更新:" + str(count)) + actions.clear() if len(actions) > 0: for success, info in helpers.parallel_bulk( DIDATA_DEV_ELASTIC_CLIENT.elastic_client, @@ -81,12 +88,12 @@ def sync(province): chunk_size=1000 # 批量大小 ): if not success: - print(f"索引失败: {info}") + logger.error(f"索引失败: {info}") count += len(actions) - print("更新:" + str(count)) + logger.info("更新:" + str(count)) actions.clear() #pd.DataFrame(errors)[['registered_address', 'company_name', 'province', 'city', 'district', 'gaode_location']].to_csv(f'{province}经纬度错误地址.csv', index=False) - log.info(f"完成同步{province}企业数据的居委村委数据") + logger.info(f"完成同步{province}企业数据的居委村委数据") # 恢复索引刷新 DIDATA_DEV_ELASTIC_CLIENT.elastic_client.indices.put_settings( index="didata_company",