diff --git a/dags/business/business/community/city_level_and_price_rating.py b/dags/business/business/community/city_level_and_price_rating.py new file mode 100644 index 0000000..234dc1e --- /dev/null +++ b/dags/business/business/community/city_level_and_price_rating.py @@ -0,0 +1,113 @@ +""" +城市等级及城市价格等级计算 +""" +from goojia_utils import ElasticSearchClient, AmapClient +import pandas as pd +from sklearn.mixture import GaussianMixture +from sklearn.preprocessing import QuantileTransformer +import logging +from elasticsearch import helpers + +es206_client = ElasticSearchClient(host='192.168.0.206') +amap_client = AmapClient() +log = logging.getLogger(__name__) +path = "/opt/airflow/dags/repo/dags/business/business/community/data" + +new_df = pd.read_excel(path + '/城市等级及城市价格等级信息.xlsx') +community_summary_df = es206_client.scan_by_sql_to_data_frame( + 'select city, region, name, consumption_potential from community_summary') + +merge_df = pd.merge(community_summary_df, new_df, how='left', on=['city']) +city_grade = ['一线', '新一线', '二线', '三四线', '五线'] +rating = [1, 2, 3, 4, 5] +index_name = "community_summary" + + +def filter_min_max(community_summary_df, source): + community_summary_df['normal_consumption_potential'] = QuantileTransformer( + output_distribution="normal").fit_transform(community_summary_df[[source]]) + return community_summary_df + + +def gmm_train(total_rate, consumption_df, source, target): + # 高斯混合模型 + gmm = GaussianMixture(n_components=total_rate, covariance_type='tied', max_iter=200, n_init=10).fit( + consumption_df[[source]]) + consumption_df.loc[:, 'cluster'] = gmm.predict(consumption_df[[source]]) + consumption_df.loc[:, 'price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean') + consumption_df.loc[:, target] = consumption_df['price_mean_cluster'].rank(method='dense', ascending=False).astype( + int) + consumption_df = consumption_df.drop(columns=['cluster', 'price_mean_cluster'], axis=1) + return consumption_df + + +log.info("开始按城市等级评级...") +merge_df = merge_df[merge_df['consumption_potential'].notna()].copy() +merge_df['consumption_potential_city_grade'] = merge_df['consumption_potential'] * merge_df['city_grade_ratio'] +merge_df['consumption_potential_city_grade'].fillna(merge_df['consumption_potential'], inplace=True) + +city_grade_ret_df = pd.DataFrame() +for grade in city_grade: + temp_df = merge_df[merge_df['city_grade'] == grade].copy() + log.info(f"当前城市等级{grade}, 评级数量{len(temp_df)}") + temp_df = filter_min_max(temp_df, 'consumption_potential_city_grade') + source = "normal_consumption_potential" + target = "city_level_rating" + temp_df = gmm_train(5, temp_df, source, target) + city_grade_ret_df = pd.concat([city_grade_ret_df, temp_df]) +log.info("完成按城市等级评级...") + +log.info("开始按城市价格等级评级") +merge_df = merge_df[merge_df['consumption_potential'].notna()].copy() +merge_df['consumption_potential_rating'] = merge_df['consumption_potential'] * merge_df['rating_ratio'] +merge_df['consumption_potential_rating'].fillna(merge_df['consumption_potential'], inplace=True) + +rating_ret_df = pd.DataFrame() +for grade in rating: + temp_df = merge_df[merge_df['rating'] == grade].copy() + log.info(f"当前城市价格等级{grade}, 评级数量{len(temp_df)}") + temp_df = filter_min_max(temp_df, 'consumption_potential_rating') + source = "normal_consumption_potential" + target = "city_price_level_rating" + temp_df = gmm_train(5, temp_df, source, target) + rating_ret_df = pd.concat([rating_ret_df, temp_df]) +log.info("完成按城市价格等级评级") + +city_grade_ret_df = city_grade_ret_df[['_id', 'city', 'region', 'name', 'city_grade', 'city_level_rating']] +rating_ret_df = rating_ret_df[['_id', 'city', 'region', 'name', 'rating', 'city_price_level_rating']] + +aim_df = pd.merge(city_grade_ret_df, rating_ret_df, how='outer', on=['_id', 'city', 'region', 'name']) + +log.info("开始写入ES206索引{index_name}") +actions = list() +count = 0 +for index, item in aim_df.iterrows(): + row = dict() + _id = str(item["_id"]) + if pd.notna(item['city_grade']): + row['city_level'] = item['city_grade'] + if pd.notna(item['city_level_rating']): + row['city_level_rating'] = item['city_level_rating'] + if pd.notna(item['rating']): + row['city_price_level'] = item['rating'] + if pd.notna(item['city_price_level_rating']): + row['city_price_level_rating'] = item['city_price_level_rating'] + + action = {'_op_type': 'update', + '_index': index_name, + '_id': _id, # 请勿忘记ID + 'doc': row, + "_retry_on_conflict": 5 + } + actions.append(action) + if len(actions) == 5000: + helpers.bulk(client=es206_client.elastic_client, actions=actions) + count += len(actions) + actions.clear() + log.info("评级中: 当前已评级{}".format(count)) +if len(actions) > 0: + helpers.bulk(client=es206_client.elastic_client, actions=actions) + count += len(actions) + actions.clear() +log.info("评级完成,共更新: {}".format(count)) +log.info("完成写入ES206索引{index_name}") \ No newline at end of file diff --git a/dags/business/business/community/data/城市等级及城市价格等级信息.xlsx b/dags/business/business/community/data/城市等级及城市价格等级信息.xlsx new file mode 100644 index 0000000..0c4e03b Binary files /dev/null and b/dags/business/business/community/data/城市等级及城市价格等级信息.xlsx differ