From e079a204720fdcc93df8de4cf6884ac67f2bb40c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 18 Jun 2025 10:50:02 +0800 Subject: [PATCH 1/4] =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/company/company/fill_company_comm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dags/company/company/fill_company_comm.py b/dags/company/company/fill_company_comm.py index e5a04b4..ef84cf8 100644 --- a/dags/company/company/fill_company_comm.py +++ b/dags/company/company/fill_company_comm.py @@ -28,6 +28,7 @@ def sync(province): count = 0 errors = list() for index, item in shape_df[shape_df['shape'].notna()].iterrows(): + print(index) ret = DIDATA_DEV_ELASTIC_CLIENT.fetch_all_geo_points('didata_enterprise', item['shape']['coordinates'][0], field='gaode_location') for info in ret: source = info['_source'] From e01efaca405abe197fc46f049ba5870f8502b658 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 18 Jun 2025 10:58:27 +0800 Subject: [PATCH 2/4] =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/company/company/fill_company_comm.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/dags/company/company/fill_company_comm.py b/dags/company/company/fill_company_comm.py index ef84cf8..122d472 100644 --- a/dags/company/company/fill_company_comm.py +++ b/dags/company/company/fill_company_comm.py @@ -5,14 +5,16 @@ from goojia_utils import DIDATA_DEV_ELASTIC_CLIENT, areautils import pandas as pd from elasticsearch import helpers import logging -log = logging.getLogger(__name__) +# 配置日志 +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +logger = logging.getLogger(__name__) def sync(province): - log.info(f"开始同步{province}企业数据的居委村委数据") + logger.info(f"开始同步{province}企业数据的居委村委数据") shape_df = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame( f"select city_name, area_name, street_name, name, shape from community where province_name='{province}'") if len(shape_df) == 0 or 'shape' not in shape_df.columns or len(shape_df[shape_df['shape'].notna()]) == 0: - log.info(f"{province}无围栏数据") + logger.info(f"{province}无围栏数据") return # 先临时禁用索引刷新 DIDATA_DEV_ELASTIC_CLIENT.elastic_client.indices.put_settings( @@ -28,7 +30,7 @@ def sync(province): count = 0 errors = list() for index, item in shape_df[shape_df['shape'].notna()].iterrows(): - print(index) + logger.info(index) ret = DIDATA_DEV_ELASTIC_CLIENT.fetch_all_geo_points('didata_enterprise', item['shape']['coordinates'][0], field='gaode_location') for info in ret: source = info['_source'] @@ -68,9 +70,9 @@ def sync(province): chunk_size=1000 # 批量大小 ): if not success: - print(f"索引失败: {info}") + logger.info(f"索引失败: {info}") count += len(actions) - print("更新:" + str(count)) + logger.info("更新:" + str(count)) actions.clear() #pd.DataFrame(errors)[['registered_address', 'company_name', 'province', 'city', 'district', 'gaode_location']].to_csv(f'{province}经纬度错误地址.csv', index=False) if len(actions) > 0: @@ -82,12 +84,12 @@ def sync(province): chunk_size=1000 # 批量大小 ): if not success: - print(f"索引失败: {info}") + logger.error(f"索引失败: {info}") count += len(actions) - print("更新:" + str(count)) + logger.info("更新:" + str(count)) actions.clear() #pd.DataFrame(errors)[['registered_address', 'company_name', 'province', 'city', 'district', 'gaode_location']].to_csv(f'{province}经纬度错误地址.csv', index=False) - log.info(f"完成同步{province}企业数据的居委村委数据") + logger.info(f"完成同步{province}企业数据的居委村委数据") # 恢复索引刷新 DIDATA_DEV_ELASTIC_CLIENT.elastic_client.indices.put_settings( index="didata_company", From 8b9ff84b64f77c0dff6ef7f8f01328a78e9f2b40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 18 Jun 2025 11:20:40 +0800 Subject: [PATCH 3/4] =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/company/company/fill_company_comm.py | 31 +++++++++++++---------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/dags/company/company/fill_company_comm.py b/dags/company/company/fill_company_comm.py index 122d472..65acd1d 100644 --- a/dags/company/company/fill_company_comm.py +++ b/dags/company/company/fill_company_comm.py @@ -16,6 +16,7 @@ def sync(province): if len(shape_df) == 0 or 'shape' not in shape_df.columns or len(shape_df[shape_df['shape'].notna()]) == 0: logger.info(f"{province}无围栏数据") return + logger.info(f"已加载{len(shape_df)}条围栏") # 先临时禁用索引刷新 DIDATA_DEV_ELASTIC_CLIENT.elastic_client.indices.put_settings( index="didata_company", @@ -30,9 +31,11 @@ def sync(province): count = 0 errors = list() for index, item in shape_df[shape_df['shape'].notna()].iterrows(): - logger.info(index) + logger.info(f"开始查找{item['city_name'], item['area_name'], item['name']}下企业数据") ret = DIDATA_DEV_ELASTIC_CLIENT.fetch_all_geo_points('didata_enterprise', item['shape']['coordinates'][0], field='gaode_location') + com_count = 0 for info in ret: + com_count += 1 source = info['_source'] if 'district' in source and (source['district'] != item['area_name']): errors.append(source) @@ -61,20 +64,20 @@ def sync(province): } actions.append(action) - if len(actions) == 10000: - for success, info in helpers.parallel_bulk( - DIDATA_DEV_ELASTIC_CLIENT.elastic_client, - actions, - thread_count=5, # 并行线程数 - refresh=False, - chunk_size=1000 # 批量大小 - ): - if not success: - logger.info(f"索引失败: {info}") - count += len(actions) - logger.info("更新:" + str(count)) - actions.clear() #pd.DataFrame(errors)[['registered_address', 'company_name', 'province', 'city', 'district', 'gaode_location']].to_csv(f'{province}经纬度错误地址.csv', index=False) + logger.info(f"已查找到该围栏下{com_count}个企业数据") + for success, info in helpers.parallel_bulk( + DIDATA_DEV_ELASTIC_CLIENT.elastic_client, + actions, + thread_count=5, # 并行线程数 + refresh=False, + chunk_size=1000 # 批量大小 + ): + if not success: + logger.info(f"索引失败: {info}") + count += len(actions) + logger.info("更新:" + str(count)) + actions.clear() if len(actions) > 0: for success, info in helpers.parallel_bulk( DIDATA_DEV_ELASTIC_CLIENT.elastic_client, From b3fbf74e5c02a81446a36264e2509b0f139f336e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 18 Jun 2025 11:23:55 +0800 Subject: [PATCH 4/4] =?UTF-8?q?=E6=9A=82=E6=97=B6=E6=B3=A8=E9=87=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/company/company/fill_company_comm.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dags/company/company/fill_company_comm.py b/dags/company/company/fill_company_comm.py index 65acd1d..3d447b2 100644 --- a/dags/company/company/fill_company_comm.py +++ b/dags/company/company/fill_company_comm.py @@ -22,7 +22,8 @@ def sync(province): index="didata_company", body={"index": {"refresh_interval": "-1"}} ) - ids = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame(f"select company_name, committee from didata_company where province='{province}'") + #ids = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame(f"select company_name, committee from didata_company where province='{province}'") + ids = pd.DataFrame({}) if len(ids) > 0: company_ids = ids['_id'].to_list() else: