新增同步字段
This commit is contained in:
@@ -0,0 +1,94 @@
|
||||
"""
|
||||
百分位计算 1 最好 100最差
|
||||
"""
|
||||
from goojia_utils import ElasticSearchClient
|
||||
import pandas as pd
|
||||
import logging
|
||||
from elasticsearch import helpers
|
||||
import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
index_name = "community_summary"
|
||||
|
||||
|
||||
def calc_percentile(city=None):
|
||||
"""
|
||||
计算消费等级
|
||||
"""
|
||||
sql = "select id, city, region, name, avg_price, total_price from community_summary"
|
||||
if pd.notna(city):
|
||||
sql += " where city='" + city + "'"
|
||||
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
|
||||
|
||||
consumption_aim_df = community_summary_df[['id', 'city', 'avg_price', 'total_price']]
|
||||
|
||||
number = []
|
||||
for i in range(1, 101):
|
||||
number.append(i / 100)
|
||||
|
||||
percents_actions = list()
|
||||
for city_now, item in consumption_aim_df.groupby(by=['city']).describe(percentiles=number).iterrows():
|
||||
for i in range(1, 101):
|
||||
if str(i) + '%' in item['total_price']:
|
||||
if pd.notna((item['total_price'])[str(i) + '%']):
|
||||
action = {'city': city_now, 'type': 'total_price',
|
||||
'price': int((item['total_price'])[str(i) + '%'] / 10000), 'percent': i}
|
||||
else:
|
||||
if pd.notna((item['total_price'])[str(i) + '.0%']):
|
||||
action = {'city': city_now, 'type': 'total_price',
|
||||
'price': int((item['total_price'])[str(i) + '.0%'] / 10000), 'percent': i}
|
||||
percents_actions.append(action)
|
||||
for i in range(1, 101):
|
||||
if str(i) + '%' in item['avg_price']:
|
||||
if pd.notna((item['avg_price'])[str(i) + '%']):
|
||||
action = {'city': city_now, 'type': 'avg_price', 'price': (item['avg_price'])[str(i) + '%'],
|
||||
'percent': i}
|
||||
else:
|
||||
if pd.notna((item['avg_price'])[str(i) + '.0%']):
|
||||
action = {'city': city_now, 'type': 'avg_price', 'price': (item['avg_price'])[str(i) + '.0%'],
|
||||
'percent': i}
|
||||
percents_actions.append(action)
|
||||
|
||||
percents_df = pd.DataFrame(percents_actions)
|
||||
|
||||
cities = consumption_aim_df['city'].unique()
|
||||
actions = list()
|
||||
count = 0
|
||||
|
||||
for city_now in cities:
|
||||
df = consumption_aim_df[consumption_aim_df['city'] == city_now].copy()
|
||||
for i in range(1, 100):
|
||||
df.loc[df['avg_price'] > percents_df[(percents_df['type'] == 'avg_price') & (percents_df['percent'] == i)].iloc[0]['price'],
|
||||
df['avg_price'] <= percents_df[(percents_df['type'] == 'avg_price') & (percents_df['percent'] == i + 1)].iloc[0]['price'], 'avg_price_percentile'] = 101 - i
|
||||
|
||||
df.loc[df['total_price'] > percents_df[(percents_df['type'] == 'total_price') & (percents_df['percent'] == i)].iloc[0]['price'],
|
||||
df['total_price'] <= percents_df[(percents_df['type'] == 'total_price') & (percents_df['percent'] == i + 1)].iloc[0]['price'], 'total_price_percentile'] = 101 - i
|
||||
|
||||
# 批量更新
|
||||
for index, item in df.iterrows():
|
||||
row = dict()
|
||||
_id = str(item["_id"])
|
||||
row['total_price_percentile'] = item['total_price_percentile']
|
||||
row['avg_price_percentile'] = item['avg_price_percentile']
|
||||
|
||||
action = {'_op_type': 'update',
|
||||
'_index': index_name,
|
||||
'_id': _id, # 请勿忘记ID
|
||||
'doc': row}
|
||||
actions.append(action)
|
||||
if len(actions) == 5000:
|
||||
helpers.bulk(client=es206_client.elastic_client, actions=actions)
|
||||
count += len(actions)
|
||||
actions.clear()
|
||||
log.info("已更新:".format(count))
|
||||
if len(actions) > 0:
|
||||
helpers.bulk(client=es206_client.elastic_client, actions=actions)
|
||||
count += len(actions)
|
||||
actions.clear()
|
||||
log.info("更新完成,共更新: {}".format(count))
|
||||
|
||||
|
||||
city = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
calc_percentile(city)
|
||||
Reference in New Issue
Block a user