优化企业开发区同步
This commit is contained in:
@@ -20,6 +20,9 @@ def sync(province):
|
||||
)
|
||||
|
||||
shape_df = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame(f"select name, shape from development_zone where province='{province}'")
|
||||
if len(shape_df) == 0 or 'shape' not in shape_df.columns or len(shape_df[shape_df['shape'].notna()]) == 0:
|
||||
logger.info(f"{province}无围栏数据")
|
||||
return
|
||||
actions = list()
|
||||
count = 0
|
||||
for index, item in shape_df[shape_df['shape'].notna()].iterrows():
|
||||
@@ -40,17 +43,18 @@ def sync(province):
|
||||
|
||||
actions.append(action)
|
||||
logger.info(f"已查找到该围栏下{com_count}个企业数据")
|
||||
for success, info in helpers.parallel_bulk(
|
||||
DIDATA_DEV_ELASTIC_CLIENT.elastic_client,
|
||||
actions,
|
||||
thread_count=5, # 并行线程数
|
||||
chunk_size=1000 # 批量大小
|
||||
):
|
||||
if not success:
|
||||
print(f"索引失败: {info}")
|
||||
count += len(actions)
|
||||
print("更新:" + str(count))
|
||||
actions.clear()
|
||||
if len(actions) > 0:
|
||||
for success, info in helpers.parallel_bulk(
|
||||
DIDATA_DEV_ELASTIC_CLIENT.elastic_client,
|
||||
actions,
|
||||
thread_count=5, # 并行线程数
|
||||
chunk_size=1000 # 批量大小
|
||||
):
|
||||
if not success:
|
||||
print(f"索引失败: {info}")
|
||||
count += len(actions)
|
||||
print("更新:" + str(count))
|
||||
actions.clear()
|
||||
if len(actions) > 0:
|
||||
for success, info in helpers.parallel_bulk(
|
||||
DIDATA_DEV_ELASTIC_CLIENT.elastic_client,
|
||||
|
||||
Reference in New Issue
Block a user