From 0727376478cff1954fc10ae3e66bbb95c1131118 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Thu, 23 Mar 2023 09:47:24 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E7=9B=B8=E5=AF=B9=E4=BB=B7?= =?UTF-8?q?=E6=A0=BC=E6=8E=92=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../community/relative_price_ranking.py | 42 +++++++++++-------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/dags/business/business/community/relative_price_ranking.py b/dags/business/business/community/relative_price_ranking.py index 9d1f67d..e576ed9 100644 --- a/dags/business/business/community/relative_price_ranking.py +++ b/dags/business/business/community/relative_price_ranking.py @@ -70,6 +70,7 @@ def build_doc(item): find_count = 0 actions = list() + def find_around_price_item(seaweed): if 'lng' not in seaweed: return @@ -78,26 +79,14 @@ def find_around_price_item(seaweed): if pd.notna(lng): city_now = seaweed['city'] around_seaweed_df = find_around_seaweed(city_now, lng, lat) + # 某些经纬度根本不在该城市下,数据问题 + if len(around_seaweed_df) == 0: + return mutex.acquire(timeout=300) global find_count find_count += 1 if find_count % 1000 == 0: log.info("已查找" + str(find_count)) - doc_info = build_doc(seaweed) - - action = { - "_op_type": "update", - "_index": index_name, - "_id": seaweed['_id'], - "doc": doc_info, - "_retry_on_conflict": 5 - } - - actions.append(action) - if len(actions) == 5000: - helpers.bulk(es206_client.elastic_client, actions) - log.info("更新:" + str(find_count)) - actions.clear() if len(around_seaweed_df) > 1: around_seaweed_df['relative_price_ranking'] = around_seaweed_df['avg_price'].rank(pct=True) @@ -117,6 +106,22 @@ def find_around_price_item(seaweed): relative_price_ranking_score = 7 seaweed['credit_detail'] = {'around_community_num': around_community_num, 'relative_price_ranking_score': relative_price_ranking_score} + + doc_info = build_doc(seaweed) + + action = { + "_op_type": "update", + "_index": index_name, + "_id": seaweed['_id'], + "doc": doc_info, + "_retry_on_conflict": 5 + } + + actions.append(action) + if len(actions) == 5000: + helpers.bulk(es206_client.elastic_client, actions) + log.info("更新:" + str(find_count)) + actions.clear() mutex.release() @@ -131,15 +136,16 @@ def sync(city=None): seaweed_list = list() seaweed_df.apply(lambda x: seaweed_list.append(dict(x)), axis=1) - with ThreadPoolExecutor(max_workers=30) as pool: + with ThreadPoolExecutor(max_workers=50) as pool: for seaweed in seaweed_list: pool.submit(find_around_price_item, seaweed) + # for seaweed in seaweed_list: + # find_around_price_item(seaweed) + if len(actions) > 0: helpers.bulk(es206_client.elastic_client, actions) actions.clear() - # for seaweed in seaweed_list: - # find_around_price_item(seaweed) log.info("完成同步相对价格排名数据...")