From b00f3a67a18b746d0efb3354c2c21eda8bebafd5 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Mon, 6 Mar 2023 15:25:47 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E4=BA=BA=E5=8F=A3=E8=A7=84?= =?UTF-8?q?=E6=A8=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../business/community/person_rating.py | 79 +++++++++++++++++++ dags/business/community_summary_dag.py | 6 +- 2 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 dags/business/business/community/person_rating.py diff --git a/dags/business/business/community/person_rating.py b/dags/business/business/community/person_rating.py new file mode 100644 index 0000000..2868bc7 --- /dev/null +++ b/dags/business/business/community/person_rating.py @@ -0,0 +1,79 @@ +""" +人口评级 +""" +from goojia_utils import ElasticSearchClient +import pandas as pd +import logging +from elasticsearch import helpers +import sys + +from sklearn.mixture import GaussianMixture + +log = logging.getLogger(__name__) + +es206_client = ElasticSearchClient(host='192.168.0.206') +index_name = "community_summary" + + +def gmm_train(total_rate, consumption_df, source, target): + # 高斯混合模型 + gmm = GaussianMixture(n_components=total_rate, covariance_type='tied', max_iter=200, n_init=10).fit(consumption_df[[source]]) + consumption_df.loc[:, 'cluster'] = gmm.predict(consumption_df[[source]]) + consumption_df.loc[:, 'price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean') + consumption_df.loc[:, target] = consumption_df['price_mean_cluster'].rank(method='dense', ascending=False).astype(int) + consumption_df = consumption_df.drop(columns=['cluster', 'price_mean_cluster'], axis=1) + return consumption_df + + +def calc_person_rating(city=None): + """ + 计算消费等级 + """ + sql = "select id, city, region, name, household_count from community_summary" + if pd.notna(city): + sql += " where city='" + city + "'" + community_summary_df = es206_client.scan_by_sql_to_data_frame(sql) + + consumption_aim_df = community_summary_df[['_id', 'city', 'household_count']] + consumption_aim_df = consumption_aim_df.dropna(subset=['household_count']) + cities = consumption_aim_df['city'].unique() + source = "household_count" + target = "person_rating" + actions = list() + count = 0 + + for city in cities: + df = consumption_aim_df[consumption_aim_df['city'] == city].copy() + if len(df) < 5: + continue + df = gmm_train(5, df, source, target) + + # 批量更新 + for index, item in df.iterrows(): + row = dict() + _id = str(item["_id"]) + row['person_rating'] = item['person_rating'] + + action = {'_op_type': 'update', + '_index': index_name, + '_id': _id, # 请勿忘记ID + 'doc': row} + actions.append(action) + if len(actions) == 5000: + helpers.bulk(client=es206_client.elastic_client, actions=actions) + count += len(actions) + actions.clear() + log.info("评级中: 当前已评级{}".format(count)) + if len(actions) > 0: + helpers.bulk(client=es206_client.elastic_client, actions=actions) + count += len(actions) + actions.clear() + log.info("评级完成,共更新: {}".format(count)) + + +def sync(city=None): + calc_person_rating(city) + + +city = sys.argv[1] if len(sys.argv) > 1 else None +sync(city) diff --git a/dags/business/community_summary_dag.py b/dags/business/community_summary_dag.py index 3125a13..84e5a19 100644 --- a/dags/business/community_summary_dag.py +++ b/dags/business/community_summary_dag.py @@ -71,6 +71,10 @@ with DAG( task_id='consumption', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/consumption.py {{dag_run.conf["city"] if "city" in dag_run.conf}}', ) + person_rating = BashOperator( + task_id='person_rating', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/person_rating.py {{dag_run.conf["city"] if "city" in dag_run.conf}}', + ) price_rating = BashOperator( task_id='price_rating', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price_rating.py {{dag_run.conf["city"] if "city" in dag_run.conf}}', @@ -92,4 +96,4 @@ with DAG( bash_command='python /opt/airflow/dags/repo/dags/business/business/community/other.py {{dag_run.conf["city"] if "city" in dag_run.conf}}', ) comm >> price >> avg_cost >> total_price >> relative_price_ranking >> estate_charge_complete >> estate_charge_ranking \ - >> consumption >> price_rating >> sell >> trade >> traffic >> business_district >> medical >> life >> education >> education2 >> extra >> other + >> consumption >> price_rating >> person_rating >> sell >> trade >> traffic >> business_district >> medical >> life >> education >> education2 >> extra >> other