优化写入禁止立即刷新
This commit is contained in:
@@ -9,6 +9,11 @@ log = logging.getLogger(__name__)
|
||||
|
||||
def sync(province):
|
||||
log.info(f"开始同步{province}企业数据的居委村委数据")
|
||||
# 先临时禁用索引刷新
|
||||
DIDATA_DEV_ELASTIC_CLIENT.elastic_client.indices.put_settings(
|
||||
index="didata_company",
|
||||
body={"index": {"refresh_interval": "-1"}}
|
||||
)
|
||||
ids = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame(f"select company_name from didata_company where province='{province}'")
|
||||
if len(ids) > 0:
|
||||
company_ids = ids['_id'].to_list()
|
||||
@@ -57,6 +62,7 @@ def sync(province):
|
||||
DIDATA_DEV_ELASTIC_CLIENT.elastic_client,
|
||||
actions,
|
||||
thread_count=5, # 并行线程数
|
||||
refresh=False,
|
||||
chunk_size=1000 # 批量大小
|
||||
):
|
||||
if not success:
|
||||
@@ -70,6 +76,7 @@ def sync(province):
|
||||
DIDATA_DEV_ELASTIC_CLIENT.elastic_client,
|
||||
actions,
|
||||
thread_count=5, # 并行线程数
|
||||
refresh=False,
|
||||
chunk_size=1000 # 批量大小
|
||||
):
|
||||
if not success:
|
||||
@@ -79,7 +86,13 @@ def sync(province):
|
||||
actions.clear()
|
||||
#pd.DataFrame(errors)[['registered_address', 'company_name', 'province', 'city', 'district', 'gaode_location']].to_csv(f'{province}经纬度错误地址.csv', index=False)
|
||||
log.info(f"完成同步{province}企业数据的居委村委数据")
|
||||
# 恢复索引刷新
|
||||
DIDATA_DEV_ELASTIC_CLIENT.elastic_client.indices.put_settings(
|
||||
index="didata_company",
|
||||
body={"index": {"refresh_interval": "5s"}}
|
||||
)
|
||||
|
||||
|
||||
for province in areautils.get_all_province():
|
||||
sync(province)
|
||||
# for province in areautils.get_all_province():
|
||||
# sync(province)
|
||||
sync('山东省')
|
||||
Reference in New Issue
Block a user