diff --git a/dags/company/company/fill_company_comm.py b/dags/company/company/fill_company_comm.py index ef84cf8..122d472 100644 --- a/dags/company/company/fill_company_comm.py +++ b/dags/company/company/fill_company_comm.py @@ -5,14 +5,16 @@ from goojia_utils import DIDATA_DEV_ELASTIC_CLIENT, areautils import pandas as pd from elasticsearch import helpers import logging -log = logging.getLogger(__name__) +# 配置日志 +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +logger = logging.getLogger(__name__) def sync(province): - log.info(f"开始同步{province}企业数据的居委村委数据") + logger.info(f"开始同步{province}企业数据的居委村委数据") shape_df = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame( f"select city_name, area_name, street_name, name, shape from community where province_name='{province}'") if len(shape_df) == 0 or 'shape' not in shape_df.columns or len(shape_df[shape_df['shape'].notna()]) == 0: - log.info(f"{province}无围栏数据") + logger.info(f"{province}无围栏数据") return # 先临时禁用索引刷新 DIDATA_DEV_ELASTIC_CLIENT.elastic_client.indices.put_settings( @@ -28,7 +30,7 @@ def sync(province): count = 0 errors = list() for index, item in shape_df[shape_df['shape'].notna()].iterrows(): - print(index) + logger.info(index) ret = DIDATA_DEV_ELASTIC_CLIENT.fetch_all_geo_points('didata_enterprise', item['shape']['coordinates'][0], field='gaode_location') for info in ret: source = info['_source'] @@ -68,9 +70,9 @@ def sync(province): chunk_size=1000 # 批量大小 ): if not success: - print(f"索引失败: {info}") + logger.info(f"索引失败: {info}") count += len(actions) - print("更新:" + str(count)) + logger.info("更新:" + str(count)) actions.clear() #pd.DataFrame(errors)[['registered_address', 'company_name', 'province', 'city', 'district', 'gaode_location']].to_csv(f'{province}经纬度错误地址.csv', index=False) if len(actions) > 0: @@ -82,12 +84,12 @@ def sync(province): chunk_size=1000 # 批量大小 ): if not success: - print(f"索引失败: {info}") + logger.error(f"索引失败: {info}") count += len(actions) - print("更新:" + str(count)) + logger.info("更新:" + str(count)) actions.clear() #pd.DataFrame(errors)[['registered_address', 'company_name', 'province', 'city', 'district', 'gaode_location']].to_csv(f'{province}经纬度错误地址.csv', index=False) - log.info(f"完成同步{province}企业数据的居委村委数据") + logger.info(f"完成同步{province}企业数据的居委村委数据") # 恢复索引刷新 DIDATA_DEV_ELASTIC_CLIENT.elastic_client.indices.put_settings( index="didata_company",