优化写入禁止立即刷新

This commit is contained in:
沈健
2025-06-06 10:29:49 +08:00
parent 20b606a0dd
commit ca1cf0f15c
+15 -2
View File
@@ -9,6 +9,11 @@ log = logging.getLogger(__name__)
def sync(province):
log.info(f"开始同步{province}企业数据的居委村委数据")
# 先临时禁用索引刷新
DIDATA_DEV_ELASTIC_CLIENT.elastic_client.indices.put_settings(
index="didata_company",
body={"index": {"refresh_interval": "-1"}}
)
ids = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame(f"select company_name from didata_company where province='{province}'")
if len(ids) > 0:
company_ids = ids['_id'].to_list()
@@ -57,6 +62,7 @@ def sync(province):
DIDATA_DEV_ELASTIC_CLIENT.elastic_client,
actions,
thread_count=5, # 并行线程数
refresh=False,
chunk_size=1000 # 批量大小
):
if not success:
@@ -70,6 +76,7 @@ def sync(province):
DIDATA_DEV_ELASTIC_CLIENT.elastic_client,
actions,
thread_count=5, # 并行线程数
refresh=False,
chunk_size=1000 # 批量大小
):
if not success:
@@ -79,7 +86,13 @@ def sync(province):
actions.clear()
#pd.DataFrame(errors)[['registered_address', 'company_name', 'province', 'city', 'district', 'gaode_location']].to_csv(f'{province}经纬度错误地址.csv', index=False)
log.info(f"完成同步{province}企业数据的居委村委数据")
# 恢复索引刷新
DIDATA_DEV_ELASTIC_CLIENT.elastic_client.indices.put_settings(
index="didata_company",
body={"index": {"refresh_interval": "5s"}}
)
for province in areautils.get_all_province():
sync(province)
# for province in areautils.get_all_province():
# sync(province)
sync('山东省')