From 958af45e794a83fb304f147b6423eed66a58d3ce Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Thu, 17 Nov 2022 15:34:09 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=8D=95=E7=8B=AC=E5=9F=8E?= =?UTF-8?q?=E5=B8=82=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/business/business/community/avg_cost.py | 30 +++++++++---------- .../business/community/consumption.py | 4 ++- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/dags/business/business/community/avg_cost.py b/dags/business/business/community/avg_cost.py index 5731179..3253335 100644 --- a/dags/business/business/community/avg_cost.py +++ b/dags/business/business/community/avg_cost.py @@ -3,7 +3,7 @@ """ import logging import pandas as pd -from goojia_utils import ElasticSearchClient, commutils, dateutils +from goojia_utils import ElasticSearchClient, commutils, dateutils, areautils from elasticsearch import helpers import numpy as np from concurrent.futures import ThreadPoolExecutor @@ -85,10 +85,12 @@ def update_status(new_df): actions.clear() -def sync(): +def sync(city=None): log.info("开始同步消费数据...") - df = es206_client.scan_by_sql_to_data_frame( - "select id, city, tag, type, price from bd_poi where price > 0 and tag in('美食;中餐厅', '美食;外国餐厅')") + query_sql = "select id, city, tag, type, price from bd_poi where price > 0 and tag in('美食;中餐厅', '美食;外国餐厅') " + if pd.notna(city): + query_sql += " and city='" + areautils.get_city_display(city) + "'" + df = es206_client.scan_by_sql_to_data_frame(query_sql) # 按百分位5-95筛选数据 cities = df['city'].unique() @@ -103,6 +105,9 @@ def sync(): update_status(new_df) seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8') + if pd.notna(city): + seaweed_df = seaweed_df[seaweed_df['city'] == city].copy() + seaweed_df = seaweed_df.reset_index(drop=True) def find_around_price_item(index, item): if 'lng' not in item: @@ -125,23 +130,18 @@ def sync(): "_op_type": "update", "_index": index_name, "_id": item['_id'], - "doc": doc_info + "doc": doc_info, + "_retry_on_conflict": 5 } actions.append(action) if len(actions) == 5000: - try: - helpers.bulk(es206_client.elastic_client, actions) - count += len(actions) - log.info("更新:" + str(count)) - except Exception as e: - log.error(e) + helpers.bulk(es206_client.elastic_client, actions) + count += len(actions) + log.info("更新:" + str(count)) actions.clear() if len(actions) > 0: - try: - helpers.bulk(es206_client.elastic_client, actions) - except Exception as e: - log.error(e) + helpers.bulk(es206_client.elastic_client, actions) actions.clear() log.info("完成同步消费数据...") diff --git a/dags/business/business/community/consumption.py b/dags/business/business/community/consumption.py index 8db03b0..36154db 100644 --- a/dags/business/business/community/consumption.py +++ b/dags/business/business/community/consumption.py @@ -99,7 +99,9 @@ def calc_consumption_rating(community_summary_df): action = {'_op_type': 'update', # 操作 index update create delete '_index': index_name, # index '_id': _id, # 请勿忘记ID - 'doc': row} + 'doc': row, + "_retry_on_conflict": 5 + } actions.append(action) if len(actions) == 5000: helpers.bulk(client=es206_client.elastic_client, actions=actions)