diff --git a/dags/company/company/fill_company_comm.py b/dags/company/company/fill_company_comm.py index ddd407f..32e76fb 100644 --- a/dags/company/company/fill_company_comm.py +++ b/dags/company/company/fill_company_comm.py @@ -9,6 +9,11 @@ log = logging.getLogger(__name__) def sync(province): log.info(f"开始同步{province}企业数据的居委村委数据") + # 先临时禁用索引刷新 + DIDATA_DEV_ELASTIC_CLIENT.elastic_client.indices.put_settings( + index="didata_company", + body={"index": {"refresh_interval": "-1"}} + ) ids = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame(f"select company_name from didata_company where province='{province}'") if len(ids) > 0: company_ids = ids['_id'].to_list() @@ -57,6 +62,7 @@ def sync(province): DIDATA_DEV_ELASTIC_CLIENT.elastic_client, actions, thread_count=5, # 并行线程数 + refresh=False, chunk_size=1000 # 批量大小 ): if not success: @@ -70,6 +76,7 @@ def sync(province): DIDATA_DEV_ELASTIC_CLIENT.elastic_client, actions, thread_count=5, # 并行线程数 + refresh=False, chunk_size=1000 # 批量大小 ): if not success: @@ -79,7 +86,13 @@ def sync(province): actions.clear() #pd.DataFrame(errors)[['registered_address', 'company_name', 'province', 'city', 'district', 'gaode_location']].to_csv(f'{province}经纬度错误地址.csv', index=False) log.info(f"完成同步{province}企业数据的居委村委数据") + # 恢复索引刷新 + DIDATA_DEV_ELASTIC_CLIENT.elastic_client.indices.put_settings( + index="didata_company", + body={"index": {"refresh_interval": "5s"}} + ) -for province in areautils.get_all_province(): - sync(province) +# for province in areautils.get_all_province(): +# sync(province) +sync('山东省') \ No newline at end of file