diff --git a/dags/business/business/community/city_level_and_price_rating.py b/dags/business/business/community/city_level_and_price_rating.py index 234dc1e..a8eb612 100644 --- a/dags/business/business/community/city_level_and_price_rating.py +++ b/dags/business/business/community/city_level_and_price_rating.py @@ -1,113 +1,121 @@ """ 城市等级及城市价格等级计算 """ -from goojia_utils import ElasticSearchClient, AmapClient +from goojia_utils import ElasticSearchClient import pandas as pd from sklearn.mixture import GaussianMixture from sklearn.preprocessing import QuantileTransformer import logging from elasticsearch import helpers - -es206_client = ElasticSearchClient(host='192.168.0.206') -amap_client = AmapClient() -log = logging.getLogger(__name__) -path = "/opt/airflow/dags/repo/dags/business/business/community/data" - -new_df = pd.read_excel(path + '/城市等级及城市价格等级信息.xlsx') -community_summary_df = es206_client.scan_by_sql_to_data_frame( - 'select city, region, name, consumption_potential from community_summary') - -merge_df = pd.merge(community_summary_df, new_df, how='left', on=['city']) -city_grade = ['一线', '新一线', '二线', '三四线', '五线'] -rating = [1, 2, 3, 4, 5] -index_name = "community_summary" +import sys -def filter_min_max(community_summary_df, source): - community_summary_df['normal_consumption_potential'] = QuantileTransformer( - output_distribution="normal").fit_transform(community_summary_df[[source]]) - return community_summary_df +def sync(): + es206_client = ElasticSearchClient(host='192.168.0.206') + log = logging.getLogger(__name__) + path = "/opt/airflow/dags/repo/dags/business/business/community/data" + new_df = pd.read_excel(path + '/城市等级及城市价格等级信息.xlsx') + community_summary_df = es206_client.scan_by_sql_to_data_frame( + 'select city, region, name, consumption_potential from community_summary') -def gmm_train(total_rate, consumption_df, source, target): - # 高斯混合模型 - gmm = GaussianMixture(n_components=total_rate, covariance_type='tied', max_iter=200, n_init=10).fit( - consumption_df[[source]]) - consumption_df.loc[:, 'cluster'] = gmm.predict(consumption_df[[source]]) - consumption_df.loc[:, 'price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean') - consumption_df.loc[:, target] = consumption_df['price_mean_cluster'].rank(method='dense', ascending=False).astype( - int) - consumption_df = consumption_df.drop(columns=['cluster', 'price_mean_cluster'], axis=1) - return consumption_df + merge_df = pd.merge(community_summary_df, new_df, how='left', on=['city']) + city_grade = ['一线', '新一线', '二线', '三四线', '五线'] + rating = [1, 2, 3, 4, 5] + index_name = "community_summary" + def filter_min_max(community_summary_df, source): + community_summary_df['normal_consumption_potential'] = QuantileTransformer( + output_distribution="normal").fit_transform(community_summary_df[[source]]) + return community_summary_df -log.info("开始按城市等级评级...") -merge_df = merge_df[merge_df['consumption_potential'].notna()].copy() -merge_df['consumption_potential_city_grade'] = merge_df['consumption_potential'] * merge_df['city_grade_ratio'] -merge_df['consumption_potential_city_grade'].fillna(merge_df['consumption_potential'], inplace=True) + def gmm_train(total_rate, consumption_df, source, target): + # 高斯混合模型 + gmm = GaussianMixture(n_components=total_rate, covariance_type='tied', max_iter=200, n_init=10).fit( + consumption_df[[source]]) + consumption_df.loc[:, 'cluster'] = gmm.predict(consumption_df[[source]]) + consumption_df.loc[:, 'price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean') + consumption_df.loc[:, target] = consumption_df['price_mean_cluster'].rank(method='dense', + ascending=False).astype( + int) + consumption_df = consumption_df.drop(columns=['cluster', 'price_mean_cluster'], axis=1) + return consumption_df -city_grade_ret_df = pd.DataFrame() -for grade in city_grade: - temp_df = merge_df[merge_df['city_grade'] == grade].copy() - log.info(f"当前城市等级{grade}, 评级数量{len(temp_df)}") - temp_df = filter_min_max(temp_df, 'consumption_potential_city_grade') - source = "normal_consumption_potential" - target = "city_level_rating" - temp_df = gmm_train(5, temp_df, source, target) - city_grade_ret_df = pd.concat([city_grade_ret_df, temp_df]) -log.info("完成按城市等级评级...") + log.info("开始按城市等级评级...") + merge_df = merge_df[merge_df['consumption_potential'].notna()].copy() + merge_df['consumption_potential_city_grade'] = merge_df['consumption_potential'] * merge_df['city_grade_ratio'] + merge_df['consumption_potential_city_grade'].fillna(merge_df['consumption_potential'], inplace=True) -log.info("开始按城市价格等级评级") -merge_df = merge_df[merge_df['consumption_potential'].notna()].copy() -merge_df['consumption_potential_rating'] = merge_df['consumption_potential'] * merge_df['rating_ratio'] -merge_df['consumption_potential_rating'].fillna(merge_df['consumption_potential'], inplace=True) + city_grade_ret_df = pd.DataFrame() + for grade in city_grade: + temp_df = merge_df[merge_df['city_grade'] == grade].copy() + log.info(f"当前城市等级{grade}, 评级数量{len(temp_df)}") + temp_df = filter_min_max(temp_df, 'consumption_potential_city_grade') + source = "normal_consumption_potential" + target = "city_level_rating" + temp_df = gmm_train(5, temp_df, source, target) + city_grade_ret_df = pd.concat([city_grade_ret_df, temp_df]) + log.info("完成按城市等级评级...") -rating_ret_df = pd.DataFrame() -for grade in rating: - temp_df = merge_df[merge_df['rating'] == grade].copy() - log.info(f"当前城市价格等级{grade}, 评级数量{len(temp_df)}") - temp_df = filter_min_max(temp_df, 'consumption_potential_rating') - source = "normal_consumption_potential" - target = "city_price_level_rating" - temp_df = gmm_train(5, temp_df, source, target) - rating_ret_df = pd.concat([rating_ret_df, temp_df]) -log.info("完成按城市价格等级评级") + log.info("开始按城市价格等级评级") + merge_df = merge_df[merge_df['consumption_potential'].notna()].copy() + merge_df['consumption_potential_rating'] = merge_df['consumption_potential'] * merge_df['rating_ratio'] + merge_df['consumption_potential_rating'].fillna(merge_df['consumption_potential'], inplace=True) -city_grade_ret_df = city_grade_ret_df[['_id', 'city', 'region', 'name', 'city_grade', 'city_level_rating']] -rating_ret_df = rating_ret_df[['_id', 'city', 'region', 'name', 'rating', 'city_price_level_rating']] + rating_ret_df = pd.DataFrame() + for grade in rating: + temp_df = merge_df[merge_df['rating'] == grade].copy() + log.info(f"当前城市价格等级{grade}, 评级数量{len(temp_df)}") + temp_df = filter_min_max(temp_df, 'consumption_potential_rating') + source = "normal_consumption_potential" + target = "city_price_level_rating" + temp_df = gmm_train(5, temp_df, source, target) + rating_ret_df = pd.concat([rating_ret_df, temp_df]) + log.info("完成按城市价格等级评级") -aim_df = pd.merge(city_grade_ret_df, rating_ret_df, how='outer', on=['_id', 'city', 'region', 'name']) + city_grade_ret_df = city_grade_ret_df[['_id', 'city', 'region', 'name', 'city_grade', 'city_level_rating']] + rating_ret_df = rating_ret_df[['_id', 'city', 'region', 'name', 'rating', 'city_price_level_rating']] -log.info("开始写入ES206索引{index_name}") -actions = list() -count = 0 -for index, item in aim_df.iterrows(): - row = dict() - _id = str(item["_id"]) - if pd.notna(item['city_grade']): - row['city_level'] = item['city_grade'] - if pd.notna(item['city_level_rating']): - row['city_level_rating'] = item['city_level_rating'] - if pd.notna(item['rating']): - row['city_price_level'] = item['rating'] - if pd.notna(item['city_price_level_rating']): - row['city_price_level_rating'] = item['city_price_level_rating'] + aim_df = pd.merge(city_grade_ret_df, rating_ret_df, how='outer', on=['_id', 'city', 'region', 'name']) - action = {'_op_type': 'update', - '_index': index_name, - '_id': _id, # 请勿忘记ID - 'doc': row, - "_retry_on_conflict": 5 - } - actions.append(action) - if len(actions) == 5000: + log.info("开始写入ES206索引{index_name}") + actions = list() + count = 0 + + city_grade_dict = {'一线': 1, '新一线': 2, '二线': 3, '三四线': 4, '五线': 5} + for index, item in aim_df.iterrows(): + row = dict() + _id = str(item["_id"]) + if pd.notna(item['city_grade']): + row['city_level'] = city_grade_dict[item['city_grade']] + if pd.notna(item['city_level_rating']): + row['city_level_rating'] = item['city_level_rating'] + if pd.notna(item['rating']): + row['city_price_level'] = item['rating'] + if pd.notna(item['city_price_level_rating']): + row['city_price_level_rating'] = item['city_price_level_rating'] + + action = {'_op_type': 'update', + '_index': index_name, + '_id': _id, # 请勿忘记ID + 'doc': row, + "_retry_on_conflict": 5 + } + actions.append(action) + if len(actions) == 5000: + helpers.bulk(client=es206_client.elastic_client, actions=actions) + count += len(actions) + actions.clear() + log.info("评级中: 当前已评级{}".format(count)) + if len(actions) > 0: helpers.bulk(client=es206_client.elastic_client, actions=actions) count += len(actions) actions.clear() - log.info("评级中: 当前已评级{}".format(count)) -if len(actions) > 0: - helpers.bulk(client=es206_client.elastic_client, actions=actions) - count += len(actions) - actions.clear() -log.info("评级完成,共更新: {}".format(count)) -log.info("完成写入ES206索引{index_name}") \ No newline at end of file + log.info("评级完成,共更新: {}".format(count)) + log.info("完成写入ES206索引{index_name}") + + +city = sys.argv[1] if len(sys.argv) > 1 else None +# 对于该评级等级来说,不能单个城市处理,只能全量 +if pd.isna(city): + sync() diff --git a/dags/business/dag_community_summary.py b/dags/business/dag_community_summary.py index 1647480..c147e84 100644 --- a/dags/business/dag_community_summary.py +++ b/dags/business/dag_community_summary.py @@ -74,6 +74,10 @@ with DAG( task_id='consumption', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/consumption.py {{dag_run.conf["city"] if "city" in dag_run.conf}}', ) + city_level_and_price_rating = BashOperator( + task_id='city_level_and_price_rating', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/city_level_and_price_rating.py {{dag_run.conf["city"] if "city" in dag_run.conf}}', + ) credit = BashOperator( task_id='credit', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/credit.py {{dag_run.conf["city"] if "city" in dag_run.conf}}', @@ -107,4 +111,4 @@ with DAG( bash_command='python /opt/airflow/dags/repo/dags/business/business/community/other.py {{dag_run.conf["city"] if "city" in dag_run.conf}}', ) # 人均消费并行,值为空 - comm >> price >> [avg_cost, total_price, relative_price_ranking, estate_charge_complete] >> total_price_complete >> relative_price_ranking_fix >> consumption >> credit >> [estate_charge_ranking, price_rating, person_rating, percentile] >> sell >> trade >> [traffic, business_district, medical, life, education] >> education2 >> extra >> other + comm >> price >> [avg_cost, total_price, relative_price_ranking, estate_charge_complete] >> total_price_complete >> relative_price_ranking_fix >> consumption >> city_level_and_price_rating >> credit >> [estate_charge_ranking, price_rating, person_rating, percentile] >> sell >> trade >> [traffic, business_district, medical, life, education] >> education2 >> extra >> other