新增城市等级级价格等级评级
This commit is contained in:
@@ -0,0 +1,113 @@
|
||||
"""
|
||||
城市等级及城市价格等级计算
|
||||
"""
|
||||
from goojia_utils import ElasticSearchClient, AmapClient
|
||||
import pandas as pd
|
||||
from sklearn.mixture import GaussianMixture
|
||||
from sklearn.preprocessing import QuantileTransformer
|
||||
import logging
|
||||
from elasticsearch import helpers
|
||||
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
amap_client = AmapClient()
|
||||
log = logging.getLogger(__name__)
|
||||
path = "/opt/airflow/dags/repo/dags/business/business/community/data"
|
||||
|
||||
new_df = pd.read_excel(path + '/城市等级及城市价格等级信息.xlsx')
|
||||
community_summary_df = es206_client.scan_by_sql_to_data_frame(
|
||||
'select city, region, name, consumption_potential from community_summary')
|
||||
|
||||
merge_df = pd.merge(community_summary_df, new_df, how='left', on=['city'])
|
||||
city_grade = ['一线', '新一线', '二线', '三四线', '五线']
|
||||
rating = [1, 2, 3, 4, 5]
|
||||
index_name = "community_summary"
|
||||
|
||||
|
||||
def filter_min_max(community_summary_df, source):
|
||||
community_summary_df['normal_consumption_potential'] = QuantileTransformer(
|
||||
output_distribution="normal").fit_transform(community_summary_df[[source]])
|
||||
return community_summary_df
|
||||
|
||||
|
||||
def gmm_train(total_rate, consumption_df, source, target):
|
||||
# 高斯混合模型
|
||||
gmm = GaussianMixture(n_components=total_rate, covariance_type='tied', max_iter=200, n_init=10).fit(
|
||||
consumption_df[[source]])
|
||||
consumption_df.loc[:, 'cluster'] = gmm.predict(consumption_df[[source]])
|
||||
consumption_df.loc[:, 'price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean')
|
||||
consumption_df.loc[:, target] = consumption_df['price_mean_cluster'].rank(method='dense', ascending=False).astype(
|
||||
int)
|
||||
consumption_df = consumption_df.drop(columns=['cluster', 'price_mean_cluster'], axis=1)
|
||||
return consumption_df
|
||||
|
||||
|
||||
log.info("开始按城市等级评级...")
|
||||
merge_df = merge_df[merge_df['consumption_potential'].notna()].copy()
|
||||
merge_df['consumption_potential_city_grade'] = merge_df['consumption_potential'] * merge_df['city_grade_ratio']
|
||||
merge_df['consumption_potential_city_grade'].fillna(merge_df['consumption_potential'], inplace=True)
|
||||
|
||||
city_grade_ret_df = pd.DataFrame()
|
||||
for grade in city_grade:
|
||||
temp_df = merge_df[merge_df['city_grade'] == grade].copy()
|
||||
log.info(f"当前城市等级{grade}, 评级数量{len(temp_df)}")
|
||||
temp_df = filter_min_max(temp_df, 'consumption_potential_city_grade')
|
||||
source = "normal_consumption_potential"
|
||||
target = "city_level_rating"
|
||||
temp_df = gmm_train(5, temp_df, source, target)
|
||||
city_grade_ret_df = pd.concat([city_grade_ret_df, temp_df])
|
||||
log.info("完成按城市等级评级...")
|
||||
|
||||
log.info("开始按城市价格等级评级")
|
||||
merge_df = merge_df[merge_df['consumption_potential'].notna()].copy()
|
||||
merge_df['consumption_potential_rating'] = merge_df['consumption_potential'] * merge_df['rating_ratio']
|
||||
merge_df['consumption_potential_rating'].fillna(merge_df['consumption_potential'], inplace=True)
|
||||
|
||||
rating_ret_df = pd.DataFrame()
|
||||
for grade in rating:
|
||||
temp_df = merge_df[merge_df['rating'] == grade].copy()
|
||||
log.info(f"当前城市价格等级{grade}, 评级数量{len(temp_df)}")
|
||||
temp_df = filter_min_max(temp_df, 'consumption_potential_rating')
|
||||
source = "normal_consumption_potential"
|
||||
target = "city_price_level_rating"
|
||||
temp_df = gmm_train(5, temp_df, source, target)
|
||||
rating_ret_df = pd.concat([rating_ret_df, temp_df])
|
||||
log.info("完成按城市价格等级评级")
|
||||
|
||||
city_grade_ret_df = city_grade_ret_df[['_id', 'city', 'region', 'name', 'city_grade', 'city_level_rating']]
|
||||
rating_ret_df = rating_ret_df[['_id', 'city', 'region', 'name', 'rating', 'city_price_level_rating']]
|
||||
|
||||
aim_df = pd.merge(city_grade_ret_df, rating_ret_df, how='outer', on=['_id', 'city', 'region', 'name'])
|
||||
|
||||
log.info("开始写入ES206索引{index_name}")
|
||||
actions = list()
|
||||
count = 0
|
||||
for index, item in aim_df.iterrows():
|
||||
row = dict()
|
||||
_id = str(item["_id"])
|
||||
if pd.notna(item['city_grade']):
|
||||
row['city_level'] = item['city_grade']
|
||||
if pd.notna(item['city_level_rating']):
|
||||
row['city_level_rating'] = item['city_level_rating']
|
||||
if pd.notna(item['rating']):
|
||||
row['city_price_level'] = item['rating']
|
||||
if pd.notna(item['city_price_level_rating']):
|
||||
row['city_price_level_rating'] = item['city_price_level_rating']
|
||||
|
||||
action = {'_op_type': 'update',
|
||||
'_index': index_name,
|
||||
'_id': _id, # 请勿忘记ID
|
||||
'doc': row,
|
||||
"_retry_on_conflict": 5
|
||||
}
|
||||
actions.append(action)
|
||||
if len(actions) == 5000:
|
||||
helpers.bulk(client=es206_client.elastic_client, actions=actions)
|
||||
count += len(actions)
|
||||
actions.clear()
|
||||
log.info("评级中: 当前已评级{}".format(count))
|
||||
if len(actions) > 0:
|
||||
helpers.bulk(client=es206_client.elastic_client, actions=actions)
|
||||
count += len(actions)
|
||||
actions.clear()
|
||||
log.info("评级完成,共更新: {}".format(count))
|
||||
log.info("完成写入ES206索引{index_name}")
|
||||
Binary file not shown.
Reference in New Issue
Block a user