From f2369687124cc60d776e54a946bdb8d96632b82b Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Fri, 4 Nov 2022 16:36:20 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E7=89=88=E6=9C=AC=E5=86=B2?= =?UTF-8?q?=E7=AA=81=E5=AF=BC=E8=87=B4=E6=97=A0=E6=B3=95=E5=86=99=E5=85=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/business/business/community/education.py | 12 +++++------- dags/business/business/community/education2.py | 12 +++++------- dags/business/business/community/extra.py | 3 ++- dags/business/business/community/medical.py | 12 +++++------- dags/business/business/community/traffic.py | 12 +++++------- 5 files changed, 22 insertions(+), 29 deletions(-) diff --git a/dags/business/business/community/education.py b/dags/business/business/community/education.py index b41edd5..4af97d0 100644 --- a/dags/business/business/community/education.py +++ b/dags/business/business/community/education.py @@ -46,18 +46,16 @@ def deal(actions, item): "_op_type": "update", "_index": index_name, "_id": item['_id'], - "doc": doc_info + "doc": doc_info, + "_retry_on_conflict": 5 } mutex.acquire(timeout=60) actions.append(action) if len(actions) == 5000: - try: - helpers.bulk(es206_client.elastic_client, actions) - count += len(actions) - log.info("更新:" + str(count)) - except Exception as e: - log.error(e) + helpers.bulk(es206_client.elastic_client, actions) + count += len(actions) + log.info("更新:" + str(count)) actions.clear() mutex.release() diff --git a/dags/business/business/community/education2.py b/dags/business/business/community/education2.py index 71842a6..9d3787c 100644 --- a/dags/business/business/community/education2.py +++ b/dags/business/business/community/education2.py @@ -58,18 +58,16 @@ def deal(actions, item): "_op_type": "update", "_index": index_name, "_id": item['_id'], - "doc": doc_info + "doc": doc_info, + "_retry_on_conflict": 5 } mutex.acquire(timeout=60) actions.append(action) if len(actions) == 5000: - try: - helpers.bulk(es206_client.elastic_client, actions) - count += len(actions) - log.info("更新:" + str(count)) - except Exception as e: - log.error(e) + helpers.bulk(es206_client.elastic_client, actions) + count += len(actions) + log.info("更新:" + str(count)) actions.clear() mutex.release() diff --git a/dags/business/business/community/extra.py b/dags/business/business/community/extra.py index b9a3a4a..4ddad24 100644 --- a/dags/business/business/community/extra.py +++ b/dags/business/business/community/extra.py @@ -95,7 +95,8 @@ def save(actions, item): "_op_type": "update", "_index": index_name, "_id": item['_id'], - "doc": doc_info + "doc": doc_info, + "_retry_on_conflict": 5 } mutex.acquire(timeout=60) actions.append(action) diff --git a/dags/business/business/community/medical.py b/dags/business/business/community/medical.py index 79d8bb4..832a4bf 100644 --- a/dags/business/business/community/medical.py +++ b/dags/business/business/community/medical.py @@ -46,18 +46,16 @@ def deal(actions, item): "_op_type": "update", "_index": index_name, "_id": item['_id'], - "doc": doc_info + "doc": doc_info, + "_retry_on_conflict": 5 } mutex.acquire(timeout=60) actions.append(action) if len(actions) == 5000: - try: - helpers.bulk(es206_client.elastic_client, actions) - count += len(actions) - log.info("更新:" + str(count)) - except Exception as e: - log.error(e) + helpers.bulk(es206_client.elastic_client, actions) + count += len(actions) + log.info("更新:" + str(count)) actions.clear() mutex.release() diff --git a/dags/business/business/community/traffic.py b/dags/business/business/community/traffic.py index c22dd51..19cf40d 100644 --- a/dags/business/business/community/traffic.py +++ b/dags/business/business/community/traffic.py @@ -55,18 +55,16 @@ def deal(actions, item): "_op_type": "update", "_index": index_name, "_id": item['_id'], - "doc": doc_info + "doc": doc_info, + "_retry_on_conflict": 5 } mutex.acquire(timeout=60) actions.append(action) if len(actions) == 5000: - try: - helpers.bulk(es206_client.elastic_client, actions) - count += len(actions) - log.info("更新:" + str(count)) - except Exception as e: - log.error(e) + helpers.bulk(es206_client.elastic_client, actions) + count += len(actions) + log.info("更新:" + str(count)) actions.clear() mutex.release()