From 1605ef816564aa6c8ca18ea0d0305059fe322415 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 20 Jun 2025 13:09:37 +0800 Subject: [PATCH 01/11] =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/company/company/fill_company_comm.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/dags/company/company/fill_company_comm.py b/dags/company/company/fill_company_comm.py index 1c33890..8c8c4af 100644 --- a/dags/company/company/fill_company_comm.py +++ b/dags/company/company/fill_company_comm.py @@ -22,8 +22,8 @@ def sync(province): index="didata_company", body={"index": {"refresh_interval": "-1"}} ) - #ids = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame(f"select company_name, committee from didata_company where province='{province}'") - ids = pd.DataFrame({}) + ids = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame(f"select company_name, committee from didata_company where province='{province}'") + #ids = pd.DataFrame({}) if len(ids) > 0: company_ids = ids['_id'].to_list() else: @@ -104,8 +104,5 @@ if __name__ == '__main__': flag = False # for province in areautils.get_all_province(): # print(province) - for province in areautils.get_all_province(): - if '甘肃省' == province: - flag = True - if flag: - sync(province) \ No newline at end of file + # for province in areautils.get_all_province(): + # sync(province) \ No newline at end of file From 663157757b000ebd606adcab31f4f01e918d607c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 20 Jun 2025 13:14:21 +0800 Subject: [PATCH 02/11] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=BC=81=E4=B8=9A?= =?UTF-8?q?=E5=BC=80=E5=8F=91=E5=8C=BA=E5=90=8C=E6=AD=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/company/company/fill_company_dev.py | 39 +++++++++++++----------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/dags/company/company/fill_company_dev.py b/dags/company/company/fill_company_dev.py index ff8ad08..ddd5ea1 100644 --- a/dags/company/company/fill_company_dev.py +++ b/dags/company/company/fill_company_dev.py @@ -4,13 +4,15 @@ from elasticsearch import helpers import numpy as np import re import logging -log = logging.getLogger(__name__) +# 配置日志 +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +logger = logging.getLogger(__name__) """ 通过开发区围栏更新企业开发区 TODO """ def sync(province): - log.info(f"开始同步{province}企业数据的开发区数据") + logger.info(f"开始同步{province}企业数据的开发区数据") # 先临时禁用索引刷新 DIDATA_DEV_ELASTIC_CLIENT.elastic_client.indices.put_settings( index="didata_company", @@ -20,12 +22,13 @@ def sync(province): shape_df = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame(f"select name, shape from development_zone where province='{province}'") actions = list() count = 0 - for index, item in shape_df.iterrows(): - if pd.isna(item['shape']): - continue + for index, item in shape_df[shape_df['shape'].notna()].iterrows(): + logger.info(f"开始查找{item['name']}下企业数据") ret = DIDATA_DEV_ELASTIC_CLIENT.fetch_all_geo_points('didata_company', item['shape']['coordinates'][0], field='location') + com_count = 0 for info in ret: + com_count += 1 doc_info = {'dev': item['name']} action = { "_op_type": "update", @@ -36,18 +39,18 @@ def sync(province): } actions.append(action) - if len(actions) == 10000: - for success, info in helpers.parallel_bulk( - DIDATA_DEV_ELASTIC_CLIENT.elastic_client, - actions, - thread_count=5, # 并行线程数 - chunk_size=1000 # 批量大小 - ): - if not success: - print(f"索引失败: {info}") - count += len(actions) - print("更新:" + str(count)) - actions.clear() + logger.info(f"已查找到该围栏下{com_count}个企业数据") + for success, info in helpers.parallel_bulk( + DIDATA_DEV_ELASTIC_CLIENT.elastic_client, + actions, + thread_count=5, # 并行线程数 + chunk_size=1000 # 批量大小 + ): + if not success: + print(f"索引失败: {info}") + count += len(actions) + print("更新:" + str(count)) + actions.clear() if len(actions) > 0: for success, info in helpers.parallel_bulk( DIDATA_DEV_ELASTIC_CLIENT.elastic_client, @@ -60,7 +63,7 @@ def sync(province): count += len(actions) print("更新:" + str(count)) actions.clear() - log.info(f"完成同步{province}企业数据的开发区数据") + logger.info(f"完成同步{province}企业数据的开发区数据") # 恢复索引刷新 DIDATA_DEV_ELASTIC_CLIENT.elastic_client.indices.put_settings( index="didata_company", From d0ebcb451a09304745915e3c6587dab6612c2047 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 20 Jun 2025 13:16:24 +0800 Subject: [PATCH 03/11] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=BC=81=E4=B8=9A?= =?UTF-8?q?=E5=BC=80=E5=8F=91=E5=8C=BA=E5=90=8C=E6=AD=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/company/dag_company.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dags/company/dag_company.py b/dags/company/dag_company.py index 773a51a..b660ffc 100644 --- a/dags/company/dag_company.py +++ b/dags/company/dag_company.py @@ -17,7 +17,7 @@ with DAG( ) t2 = BashOperator( task_id='fill_commpany_dev', - bash_command='python /opt/airflow/dags/repo/dags/company/company/fill_commpany_dev.py', + bash_command='python /opt/airflow/dags/repo/dags/company/company/fill_company_dev.py', ) t3 = BashOperator( task_id='mark_right_dev', From d0e6714ecdd5db12d169d196952e5950b552fc7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 20 Jun 2025 13:18:31 +0800 Subject: [PATCH 04/11] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=BC=81=E4=B8=9A?= =?UTF-8?q?=E5=BC=80=E5=8F=91=E5=8C=BA=E5=90=8C=E6=AD=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/company/company/fill_company_dev.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dags/company/company/fill_company_dev.py b/dags/company/company/fill_company_dev.py index ddd5ea1..d131ee3 100644 --- a/dags/company/company/fill_company_dev.py +++ b/dags/company/company/fill_company_dev.py @@ -1,4 +1,4 @@ -from goojia_utils import DIDATA_DEV_ELASTIC_CLIENT, threads, geohashutils, areautils +from goojia_utils import DIDATA_DEV_ELASTIC_CLIENT, areautils import pandas as pd from elasticsearch import helpers import numpy as np From 3975d029f843d92c3809fbca7015bb00ffb202f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 20 Jun 2025 13:26:11 +0800 Subject: [PATCH 05/11] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=BC=81=E4=B8=9A?= =?UTF-8?q?=E5=BC=80=E5=8F=91=E5=8C=BA=E5=90=8C=E6=AD=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/company/company/fill_company_dev.py | 26 ++++++++++++++---------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/dags/company/company/fill_company_dev.py b/dags/company/company/fill_company_dev.py index d131ee3..d8de094 100644 --- a/dags/company/company/fill_company_dev.py +++ b/dags/company/company/fill_company_dev.py @@ -20,6 +20,9 @@ def sync(province): ) shape_df = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame(f"select name, shape from development_zone where province='{province}'") + if len(shape_df) == 0 or 'shape' not in shape_df.columns or len(shape_df[shape_df['shape'].notna()]) == 0: + logger.info(f"{province}无围栏数据") + return actions = list() count = 0 for index, item in shape_df[shape_df['shape'].notna()].iterrows(): @@ -40,17 +43,18 @@ def sync(province): actions.append(action) logger.info(f"已查找到该围栏下{com_count}个企业数据") - for success, info in helpers.parallel_bulk( - DIDATA_DEV_ELASTIC_CLIENT.elastic_client, - actions, - thread_count=5, # 并行线程数 - chunk_size=1000 # 批量大小 - ): - if not success: - print(f"索引失败: {info}") - count += len(actions) - print("更新:" + str(count)) - actions.clear() + if len(actions) > 0: + for success, info in helpers.parallel_bulk( + DIDATA_DEV_ELASTIC_CLIENT.elastic_client, + actions, + thread_count=5, # 并行线程数 + chunk_size=1000 # 批量大小 + ): + if not success: + print(f"索引失败: {info}") + count += len(actions) + print("更新:" + str(count)) + actions.clear() if len(actions) > 0: for success, info in helpers.parallel_bulk( DIDATA_DEV_ELASTIC_CLIENT.elastic_client, From 9ce31e9b56d74fcd33946cf2a777046d640e3405 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 20 Jun 2025 13:30:38 +0800 Subject: [PATCH 06/11] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=BC=81=E4=B8=9A?= =?UTF-8?q?=E5=BC=80=E5=8F=91=E5=8C=BA=E5=90=8C=E6=AD=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/company/company/fill_company_dev.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dags/company/company/fill_company_dev.py b/dags/company/company/fill_company_dev.py index d8de094..a8ddd02 100644 --- a/dags/company/company/fill_company_dev.py +++ b/dags/company/company/fill_company_dev.py @@ -48,7 +48,7 @@ def sync(province): DIDATA_DEV_ELASTIC_CLIENT.elastic_client, actions, thread_count=5, # 并行线程数 - chunk_size=1000 # 批量大小 + chunk_size=10000 # 批量大小 ): if not success: print(f"索引失败: {info}") @@ -60,7 +60,7 @@ def sync(province): DIDATA_DEV_ELASTIC_CLIENT.elastic_client, actions, thread_count=5, # 并行线程数 - chunk_size=1000 # 批量大小 + chunk_size=10000 # 批量大小 ): if not success: print(f"索引失败: {info}") From 399151ff588bee5ad9b6ecede649636646dd978b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 20 Jun 2025 14:48:16 +0800 Subject: [PATCH 07/11] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=BC=81=E4=B8=9A?= =?UTF-8?q?=E5=BC=80=E5=8F=91=E5=8C=BA=E5=90=8C=E6=AD=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/company/company/mark_right_dev.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dags/company/company/mark_right_dev.py b/dags/company/company/mark_right_dev.py index 550cd8d..cdc629f 100644 --- a/dags/company/company/mark_right_dev.py +++ b/dags/company/company/mark_right_dev.py @@ -1,7 +1,6 @@ -from goojia_utils import DIDATA_DEV_ELASTIC_CLIENT, threads, geohashutils, areautils +from goojia_utils import DIDATA_DEV_ELASTIC_CLIENT, areautils import pandas as pd from elasticsearch import helpers -import numpy as np import re import logging log = logging.getLogger(__name__) From 9d1a9a769ef0fe7c7675b8c5d593a16a3219cf7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 20 Jun 2025 14:53:54 +0800 Subject: [PATCH 08/11] =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/company/company/mark_right_dev.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dags/company/company/mark_right_dev.py b/dags/company/company/mark_right_dev.py index cdc629f..284c178 100644 --- a/dags/company/company/mark_right_dev.py +++ b/dags/company/company/mark_right_dev.py @@ -17,7 +17,7 @@ def sync(): '绥安工业开发区': '旧镇工业园', '绥安工业开发区': '大南坂开发区', '绥安工业开发区': '大南坂工业开发区', '福建龙州工业园区': '龙州工业园', '长汀经济开发区':'腾飞开发区', '福建福州金山工业园区': '金山投资区', '厦门象屿综合保税区': '厦门现代物流园', '福清融侨经济技术开发区': '融侨经济', '福建洛江经济开发区': '万安开发区'} - dev_df = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame("select province, city, district, name, alias from development_zone where province='福建省'") + dev_df = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame("select province, city, district, name, alias from development_zone") # 不要去重,有可能存在别名, 如福建云霄常山经济开发区 #dev_df = dev_df[dev_df['name'] == '福建政和经济开发区'] actions = list() From a97cf5035be3cdf48ef6a9e5d3bbe55700d4eb39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 20 Jun 2025 14:56:17 +0800 Subject: [PATCH 09/11] =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/company/company/fill_company_comm.py | 4 ++-- dags/company/dag_company.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dags/company/company/fill_company_comm.py b/dags/company/company/fill_company_comm.py index 8c8c4af..52e61f4 100644 --- a/dags/company/company/fill_company_comm.py +++ b/dags/company/company/fill_company_comm.py @@ -104,5 +104,5 @@ if __name__ == '__main__': flag = False # for province in areautils.get_all_province(): # print(province) - # for province in areautils.get_all_province(): - # sync(province) \ No newline at end of file + for province in areautils.get_all_province(): + sync(province) \ No newline at end of file diff --git a/dags/company/dag_company.py b/dags/company/dag_company.py index b660ffc..660f5bb 100644 --- a/dags/company/dag_company.py +++ b/dags/company/dag_company.py @@ -23,4 +23,4 @@ with DAG( task_id='mark_right_dev', bash_command='python /opt/airflow/dags/repo/dags/company/company/mark_right_dev.py', ) - t1 >> t2 >> t3 \ No newline at end of file + t3 \ No newline at end of file From 7880ccf8021693b62a4cf42765badcd76a886efe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 20 Jun 2025 14:57:40 +0800 Subject: [PATCH 10/11] =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/company/dag_company.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/dags/company/dag_company.py b/dags/company/dag_company.py index 660f5bb..107569f 100644 --- a/dags/company/dag_company.py +++ b/dags/company/dag_company.py @@ -11,14 +11,14 @@ with DAG( catchup=True, description="企业涉农处理" ) as dag: - t1 = BashOperator( - task_id='fill_commpany_comm', - bash_command='python /opt/airflow/dags/repo/dags/company/company/fill_company_comm.py', - ) - t2 = BashOperator( - task_id='fill_commpany_dev', - bash_command='python /opt/airflow/dags/repo/dags/company/company/fill_company_dev.py', - ) + # t1 = BashOperator( + # task_id='fill_commpany_comm', + # bash_command='python /opt/airflow/dags/repo/dags/company/company/fill_company_comm.py', + # ) + # t2 = BashOperator( + # task_id='fill_commpany_dev', + # bash_command='python /opt/airflow/dags/repo/dags/company/company/fill_company_dev.py', + # ) t3 = BashOperator( task_id='mark_right_dev', bash_command='python /opt/airflow/dags/repo/dags/company/company/mark_right_dev.py', From f892dcee0ad1fe8e9791b5098f0e821ea8e46dd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 20 Jun 2025 15:07:09 +0800 Subject: [PATCH 11/11] =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/company/company/mark_right_dev.py | 27 +++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/dags/company/company/mark_right_dev.py b/dags/company/company/mark_right_dev.py index 284c178..51a4fe6 100644 --- a/dags/company/company/mark_right_dev.py +++ b/dags/company/company/mark_right_dev.py @@ -54,19 +54,20 @@ def sync(): # 福建诏安工业园区 福建省漳州市诏安工业园区丹诏大道15号 if pd.notna(dev.city): province_short = areautils.get_province(dev.city) - new_name = re.sub(province_short, '', dev.name) - if pd.notna(dev.city) and pd.notna(dev.district): - city_short = areautils.get_city_short(dev.city) - district_short = areautils.get_district_short(dev.city, dev.district) - if pd.notna(city_short) and city_short in new_name: - names.append(new_name) - if pd.notna(district_short) and district_short in new_name: - names.append(new_name) - # 福建省漳州市长泰县经济开发区林墩办事处帝君路 福建长泰经济开发区 - new_new_name = re.sub(district_short, district_short + '县', new_name) - names.append(new_new_name) - new_new_name = re.sub(district_short, district_short + '区', new_name) - names.append(new_new_name) + if pd.notna(province_short): + new_name = re.sub(province_short, '', dev.name) + if pd.notna(dev.city) and pd.notna(dev.district): + city_short = areautils.get_city_short(dev.city) + district_short = areautils.get_district_short(dev.city, dev.district) + if pd.notna(city_short) and city_short in new_name: + names.append(new_name) + if pd.notna(district_short) and district_short in new_name: + names.append(new_name) + # 福建省漳州市长泰县经济开发区林墩办事处帝君路 福建长泰经济开发区 + new_new_name = re.sub(district_short, district_short + '县', new_name) + names.append(new_new_name) + new_new_name = re.sub(district_short, district_short + '区', new_name) + names.append(new_new_name) for key in dev_alias: if dev.name == key: