优化企业开发区同步

This commit is contained in:
沈健
2025-06-20 13:14:21 +08:00
parent 405f40f0b8
commit 663157757b
+21 -18
View File
@@ -4,13 +4,15 @@ from elasticsearch import helpers
import numpy as np
import re
import logging
log = logging.getLogger(__name__)
# 配置日志
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)
"""
通过开发区围栏更新企业开发区 TODO
"""
def sync(province):
log.info(f"开始同步{province}企业数据的开发区数据")
logger.info(f"开始同步{province}企业数据的开发区数据")
# 先临时禁用索引刷新
DIDATA_DEV_ELASTIC_CLIENT.elastic_client.indices.put_settings(
index="didata_company",
@@ -20,12 +22,13 @@ def sync(province):
shape_df = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame(f"select name, shape from development_zone where province='{province}'")
actions = list()
count = 0
for index, item in shape_df.iterrows():
if pd.isna(item['shape']):
continue
for index, item in shape_df[shape_df['shape'].notna()].iterrows():
logger.info(f"开始查找{item['name']}下企业数据")
ret = DIDATA_DEV_ELASTIC_CLIENT.fetch_all_geo_points('didata_company', item['shape']['coordinates'][0],
field='location')
com_count = 0
for info in ret:
com_count += 1
doc_info = {'dev': item['name']}
action = {
"_op_type": "update",
@@ -36,18 +39,18 @@ def sync(province):
}
actions.append(action)
if len(actions) == 10000:
for success, info in helpers.parallel_bulk(
DIDATA_DEV_ELASTIC_CLIENT.elastic_client,
actions,
thread_count=5, # 并行线程数
chunk_size=1000 # 批量大小
):
if not success:
print(f"索引失败: {info}")
count += len(actions)
print("更新:" + str(count))
actions.clear()
logger.info(f"已查找到该围栏下{com_count}个企业数据")
for success, info in helpers.parallel_bulk(
DIDATA_DEV_ELASTIC_CLIENT.elastic_client,
actions,
thread_count=5, # 并行线程数
chunk_size=1000 # 批量大小
):
if not success:
print(f"索引失败: {info}")
count += len(actions)
print("更新:" + str(count))
actions.clear()
if len(actions) > 0:
for success, info in helpers.parallel_bulk(
DIDATA_DEV_ELASTIC_CLIENT.elastic_client,
@@ -60,7 +63,7 @@ def sync(province):
count += len(actions)
print("更新:" + str(count))
actions.clear()
log.info(f"完成同步{province}企业数据的开发区数据")
logger.info(f"完成同步{province}企业数据的开发区数据")
# 恢复索引刷新
DIDATA_DEV_ELASTIC_CLIENT.elastic_client.indices.put_settings(
index="didata_company",