优化企业开发区同步
This commit is contained in:
@@ -4,13 +4,15 @@ from elasticsearch import helpers
|
||||
import numpy as np
|
||||
import re
|
||||
import logging
|
||||
log = logging.getLogger(__name__)
|
||||
# 配置日志
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
"""
|
||||
通过开发区围栏更新企业开发区 TODO
|
||||
"""
|
||||
def sync(province):
|
||||
log.info(f"开始同步{province}企业数据的开发区数据")
|
||||
logger.info(f"开始同步{province}企业数据的开发区数据")
|
||||
# 先临时禁用索引刷新
|
||||
DIDATA_DEV_ELASTIC_CLIENT.elastic_client.indices.put_settings(
|
||||
index="didata_company",
|
||||
@@ -20,12 +22,13 @@ def sync(province):
|
||||
shape_df = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame(f"select name, shape from development_zone where province='{province}'")
|
||||
actions = list()
|
||||
count = 0
|
||||
for index, item in shape_df.iterrows():
|
||||
if pd.isna(item['shape']):
|
||||
continue
|
||||
for index, item in shape_df[shape_df['shape'].notna()].iterrows():
|
||||
logger.info(f"开始查找{item['name']}下企业数据")
|
||||
ret = DIDATA_DEV_ELASTIC_CLIENT.fetch_all_geo_points('didata_company', item['shape']['coordinates'][0],
|
||||
field='location')
|
||||
com_count = 0
|
||||
for info in ret:
|
||||
com_count += 1
|
||||
doc_info = {'dev': item['name']}
|
||||
action = {
|
||||
"_op_type": "update",
|
||||
@@ -36,18 +39,18 @@ def sync(province):
|
||||
}
|
||||
|
||||
actions.append(action)
|
||||
if len(actions) == 10000:
|
||||
for success, info in helpers.parallel_bulk(
|
||||
DIDATA_DEV_ELASTIC_CLIENT.elastic_client,
|
||||
actions,
|
||||
thread_count=5, # 并行线程数
|
||||
chunk_size=1000 # 批量大小
|
||||
):
|
||||
if not success:
|
||||
print(f"索引失败: {info}")
|
||||
count += len(actions)
|
||||
print("更新:" + str(count))
|
||||
actions.clear()
|
||||
logger.info(f"已查找到该围栏下{com_count}个企业数据")
|
||||
for success, info in helpers.parallel_bulk(
|
||||
DIDATA_DEV_ELASTIC_CLIENT.elastic_client,
|
||||
actions,
|
||||
thread_count=5, # 并行线程数
|
||||
chunk_size=1000 # 批量大小
|
||||
):
|
||||
if not success:
|
||||
print(f"索引失败: {info}")
|
||||
count += len(actions)
|
||||
print("更新:" + str(count))
|
||||
actions.clear()
|
||||
if len(actions) > 0:
|
||||
for success, info in helpers.parallel_bulk(
|
||||
DIDATA_DEV_ELASTIC_CLIENT.elastic_client,
|
||||
@@ -60,7 +63,7 @@ def sync(province):
|
||||
count += len(actions)
|
||||
print("更新:" + str(count))
|
||||
actions.clear()
|
||||
log.info(f"完成同步{province}企业数据的开发区数据")
|
||||
logger.info(f"完成同步{province}企业数据的开发区数据")
|
||||
# 恢复索引刷新
|
||||
DIDATA_DEV_ELASTIC_CLIENT.elastic_client.indices.put_settings(
|
||||
index="didata_company",
|
||||
|
||||
Reference in New Issue
Block a user