From bdc9a3c62a681b33fac628a9db82027d673b1036 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Tue, 25 Apr 2023 14:43:14 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=90=8C=E6=AD=A5=E5=AD=97?= =?UTF-8?q?=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../business/business/community/percentile.py | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 dags/business/business/community/percentile.py diff --git a/dags/business/business/community/percentile.py b/dags/business/business/community/percentile.py new file mode 100644 index 0000000..8ada251 --- /dev/null +++ b/dags/business/business/community/percentile.py @@ -0,0 +1,94 @@ +""" +百分位计算 1 最好 100最差 +""" +from goojia_utils import ElasticSearchClient +import pandas as pd +import logging +from elasticsearch import helpers +import sys + +log = logging.getLogger(__name__) + +es206_client = ElasticSearchClient(host='192.168.0.206') +index_name = "community_summary" + + +def calc_percentile(city=None): + """ + 计算消费等级 + """ + sql = "select id, city, region, name, avg_price, total_price from community_summary" + if pd.notna(city): + sql += " where city='" + city + "'" + community_summary_df = es206_client.scan_by_sql_to_data_frame(sql) + + consumption_aim_df = community_summary_df[['id', 'city', 'avg_price', 'total_price']] + + number = [] + for i in range(1, 101): + number.append(i / 100) + + percents_actions = list() + for city_now, item in consumption_aim_df.groupby(by=['city']).describe(percentiles=number).iterrows(): + for i in range(1, 101): + if str(i) + '%' in item['total_price']: + if pd.notna((item['total_price'])[str(i) + '%']): + action = {'city': city_now, 'type': 'total_price', + 'price': int((item['total_price'])[str(i) + '%'] / 10000), 'percent': i} + else: + if pd.notna((item['total_price'])[str(i) + '.0%']): + action = {'city': city_now, 'type': 'total_price', + 'price': int((item['total_price'])[str(i) + '.0%'] / 10000), 'percent': i} + percents_actions.append(action) + for i in range(1, 101): + if str(i) + '%' in item['avg_price']: + if pd.notna((item['avg_price'])[str(i) + '%']): + action = {'city': city_now, 'type': 'avg_price', 'price': (item['avg_price'])[str(i) + '%'], + 'percent': i} + else: + if pd.notna((item['avg_price'])[str(i) + '.0%']): + action = {'city': city_now, 'type': 'avg_price', 'price': (item['avg_price'])[str(i) + '.0%'], + 'percent': i} + percents_actions.append(action) + + percents_df = pd.DataFrame(percents_actions) + + cities = consumption_aim_df['city'].unique() + actions = list() + count = 0 + + for city_now in cities: + df = consumption_aim_df[consumption_aim_df['city'] == city_now].copy() + for i in range(1, 100): + df.loc[df['avg_price'] > percents_df[(percents_df['type'] == 'avg_price') & (percents_df['percent'] == i)].iloc[0]['price'], + df['avg_price'] <= percents_df[(percents_df['type'] == 'avg_price') & (percents_df['percent'] == i + 1)].iloc[0]['price'], 'avg_price_percentile'] = 101 - i + + df.loc[df['total_price'] > percents_df[(percents_df['type'] == 'total_price') & (percents_df['percent'] == i)].iloc[0]['price'], + df['total_price'] <= percents_df[(percents_df['type'] == 'total_price') & (percents_df['percent'] == i + 1)].iloc[0]['price'], 'total_price_percentile'] = 101 - i + + # 批量更新 + for index, item in df.iterrows(): + row = dict() + _id = str(item["_id"]) + row['total_price_percentile'] = item['total_price_percentile'] + row['avg_price_percentile'] = item['avg_price_percentile'] + + action = {'_op_type': 'update', + '_index': index_name, + '_id': _id, # 请勿忘记ID + 'doc': row} + actions.append(action) + if len(actions) == 5000: + helpers.bulk(client=es206_client.elastic_client, actions=actions) + count += len(actions) + actions.clear() + log.info("已更新:".format(count)) + if len(actions) > 0: + helpers.bulk(client=es206_client.elastic_client, actions=actions) + count += len(actions) + actions.clear() + log.info("更新完成,共更新: {}".format(count)) + + +city = sys.argv[1] if len(sys.argv) > 1 else None +calc_percentile(city)