新增城市等级级价格等级评级

This commit is contained in:
shenjian@fangjia.com
2023-05-15 15:14:19 +08:00
parent 0077622e63
commit 95462de314
2 changed files with 113 additions and 0 deletions
@@ -0,0 +1,113 @@
"""
城市等级及城市价格等级计算
"""
from goojia_utils import ElasticSearchClient, AmapClient
import pandas as pd
from sklearn.mixture import GaussianMixture
from sklearn.preprocessing import QuantileTransformer
import logging
from elasticsearch import helpers
es206_client = ElasticSearchClient(host='192.168.0.206')
amap_client = AmapClient()
log = logging.getLogger(__name__)
path = "/opt/airflow/dags/repo/dags/business/business/community/data"
new_df = pd.read_excel(path + '/城市等级及城市价格等级信息.xlsx')
community_summary_df = es206_client.scan_by_sql_to_data_frame(
'select city, region, name, consumption_potential from community_summary')
merge_df = pd.merge(community_summary_df, new_df, how='left', on=['city'])
city_grade = ['一线', '新一线', '二线', '三四线', '五线']
rating = [1, 2, 3, 4, 5]
index_name = "community_summary"
def filter_min_max(community_summary_df, source):
community_summary_df['normal_consumption_potential'] = QuantileTransformer(
output_distribution="normal").fit_transform(community_summary_df[[source]])
return community_summary_df
def gmm_train(total_rate, consumption_df, source, target):
# 高斯混合模型
gmm = GaussianMixture(n_components=total_rate, covariance_type='tied', max_iter=200, n_init=10).fit(
consumption_df[[source]])
consumption_df.loc[:, 'cluster'] = gmm.predict(consumption_df[[source]])
consumption_df.loc[:, 'price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean')
consumption_df.loc[:, target] = consumption_df['price_mean_cluster'].rank(method='dense', ascending=False).astype(
int)
consumption_df = consumption_df.drop(columns=['cluster', 'price_mean_cluster'], axis=1)
return consumption_df
log.info("开始按城市等级评级...")
merge_df = merge_df[merge_df['consumption_potential'].notna()].copy()
merge_df['consumption_potential_city_grade'] = merge_df['consumption_potential'] * merge_df['city_grade_ratio']
merge_df['consumption_potential_city_grade'].fillna(merge_df['consumption_potential'], inplace=True)
city_grade_ret_df = pd.DataFrame()
for grade in city_grade:
temp_df = merge_df[merge_df['city_grade'] == grade].copy()
log.info(f"当前城市等级{grade}, 评级数量{len(temp_df)}")
temp_df = filter_min_max(temp_df, 'consumption_potential_city_grade')
source = "normal_consumption_potential"
target = "city_level_rating"
temp_df = gmm_train(5, temp_df, source, target)
city_grade_ret_df = pd.concat([city_grade_ret_df, temp_df])
log.info("完成按城市等级评级...")
log.info("开始按城市价格等级评级")
merge_df = merge_df[merge_df['consumption_potential'].notna()].copy()
merge_df['consumption_potential_rating'] = merge_df['consumption_potential'] * merge_df['rating_ratio']
merge_df['consumption_potential_rating'].fillna(merge_df['consumption_potential'], inplace=True)
rating_ret_df = pd.DataFrame()
for grade in rating:
temp_df = merge_df[merge_df['rating'] == grade].copy()
log.info(f"当前城市价格等级{grade}, 评级数量{len(temp_df)}")
temp_df = filter_min_max(temp_df, 'consumption_potential_rating')
source = "normal_consumption_potential"
target = "city_price_level_rating"
temp_df = gmm_train(5, temp_df, source, target)
rating_ret_df = pd.concat([rating_ret_df, temp_df])
log.info("完成按城市价格等级评级")
city_grade_ret_df = city_grade_ret_df[['_id', 'city', 'region', 'name', 'city_grade', 'city_level_rating']]
rating_ret_df = rating_ret_df[['_id', 'city', 'region', 'name', 'rating', 'city_price_level_rating']]
aim_df = pd.merge(city_grade_ret_df, rating_ret_df, how='outer', on=['_id', 'city', 'region', 'name'])
log.info("开始写入ES206索引{index_name}")
actions = list()
count = 0
for index, item in aim_df.iterrows():
row = dict()
_id = str(item["_id"])
if pd.notna(item['city_grade']):
row['city_level'] = item['city_grade']
if pd.notna(item['city_level_rating']):
row['city_level_rating'] = item['city_level_rating']
if pd.notna(item['rating']):
row['city_price_level'] = item['rating']
if pd.notna(item['city_price_level_rating']):
row['city_price_level_rating'] = item['city_price_level_rating']
action = {'_op_type': 'update',
'_index': index_name,
'_id': _id, # 请勿忘记ID
'doc': row,
"_retry_on_conflict": 5
}
actions.append(action)
if len(actions) == 5000:
helpers.bulk(client=es206_client.elastic_client, actions=actions)
count += len(actions)
actions.clear()
log.info("评级中: 当前已评级{}".format(count))
if len(actions) > 0:
helpers.bulk(client=es206_client.elastic_client, actions=actions)
count += len(actions)
actions.clear()
log.info("评级完成,共更新: {}".format(count))
log.info("完成写入ES206索引{index_name}")