From 29d42f33d69325755b4afd56c672ecd226cb5d1e Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Thu, 23 Mar 2023 09:25:19 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E7=9B=B8=E5=AF=B9=E4=BB=B7?= =?UTF-8?q?=E6=A0=BC=E6=8E=92=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../community/relative_price_ranking.py | 59 ++++++++----------- 1 file changed, 25 insertions(+), 34 deletions(-) diff --git a/dags/business/business/community/relative_price_ranking.py b/dags/business/business/community/relative_price_ranking.py index 9f03336..9d1f67d 100644 --- a/dags/business/business/community/relative_price_ranking.py +++ b/dags/business/business/community/relative_price_ranking.py @@ -68,13 +68,9 @@ def build_doc(item): find_count = 0 - +actions = list() def find_around_price_item(seaweed): - global find_count - find_count += 1 - if find_count % 1000 == 0: - log.info("已查找" + str(find_count)) if 'lng' not in seaweed: return lng = commutils.format_lat_lng(seaweed['lng']) @@ -82,7 +78,27 @@ def find_around_price_item(seaweed): if pd.notna(lng): city_now = seaweed['city'] around_seaweed_df = find_around_seaweed(city_now, lng, lat) - mutex.acquire(timeout=60) + mutex.acquire(timeout=300) + global find_count + find_count += 1 + if find_count % 1000 == 0: + log.info("已查找" + str(find_count)) + doc_info = build_doc(seaweed) + + action = { + "_op_type": "update", + "_index": index_name, + "_id": seaweed['_id'], + "doc": doc_info, + "_retry_on_conflict": 5 + } + + actions.append(action) + if len(actions) == 5000: + helpers.bulk(es206_client.elastic_client, actions) + log.info("更新:" + str(find_count)) + actions.clear() + if len(around_seaweed_df) > 1: around_seaweed_df['relative_price_ranking'] = around_seaweed_df['avg_price'].rank(pct=True) seaweed['relative_price_ranking'] = around_seaweed_df[around_seaweed_df['id'] == seaweed['_id']].iloc[0][ @@ -119,37 +135,12 @@ def sync(city=None): for seaweed in seaweed_list: pool.submit(find_around_price_item, seaweed) + if len(actions) > 0: + helpers.bulk(es206_client.elastic_client, actions) + actions.clear() # for seaweed in seaweed_list: # find_around_price_item(seaweed) - seaweed_df = pd.DataFrame(seaweed_list) - - actions = list() - count = 0 - for index, item in seaweed_df.iterrows(): - doc_info = build_doc(item) - - action = { - "_op_type": "update", - "_index": index_name, - "_id": item['_id'], - "doc": doc_info, - "_retry_on_conflict": 5 - } - - actions.append(action) - if len(actions) == 5000: - helpers.bulk(es206_client.elastic_client, actions) - count += len(actions) - log.info("更新:" + str(count)) - actions.clear() - if len(actions) > 0: - try: - helpers.bulk(es206_client.elastic_client, actions) - except Exception as e: - log.error(e) - actions.clear() - log.info("完成同步相对价格排名数据...")