From 33174c43f38b21019cf19ecc4e1fd14ddf0020a3 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Tue, 25 Oct 2022 11:19:27 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E8=AF=84=E7=BA=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...onsumption_potential.py => consumption.py} | 71 +++++++++++++--- .../business/community/price_rating.py | 84 +++++++++++++++++++ dags/business/business/rate.py | 25 ------ dags/business/community_summary_dag.py | 11 ++- dags/business/rate_dag.py | 17 ---- requirements.txt | 3 +- 6 files changed, 153 insertions(+), 58 deletions(-) rename dags/business/business/community/{consumption_potential.py => consumption.py} (51%) create mode 100644 dags/business/business/community/price_rating.py delete mode 100644 dags/business/business/rate.py delete mode 100644 dags/business/rate_dag.py diff --git a/dags/business/business/community/consumption_potential.py b/dags/business/business/community/consumption.py similarity index 51% rename from dags/business/business/community/consumption_potential.py rename to dags/business/business/community/consumption.py index 03f5bd4..ac41bf5 100644 --- a/dags/business/business/community/consumption_potential.py +++ b/dags/business/business/community/consumption.py @@ -4,8 +4,15 @@ from goojia_utils import ElasticSearchClient import pandas as pd import numpy as np +import logging +from elasticsearch import helpers + +from sklearn.mixture import GaussianMixture + +log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') +index_name = "community_summary" def calc_consumption_potential(city): @@ -39,13 +46,11 @@ def calc_consumption_potential(city): community_summary_df['total_price_score'].fillna(0, inplace=True) community_summary_df['estate_charge_score'] = 139.1374524 * ( - community_summary_df['estate_charge'] - community_summary_df['estate_charge_mean']) / \ - community_summary_df['estate_charge_std'] + community_summary_df['estate_charge'] - community_summary_df['estate_charge_mean']) / community_summary_df['estate_charge_std'] community_summary_df['estate_charge_score'].fillna(0, inplace=True) community_summary_df['relative_price_ranking_score'] = 21.8427926 * ( - community_summary_df['relative_price_ranking'] - community_summary_df['relative_price_ranking_mean']) / \ - community_summary_df['relative_price_ranking_std'] + community_summary_df['relative_price_ranking'] - community_summary_df['relative_price_ranking_mean']) / community_summary_df['relative_price_ranking_std'] community_summary_df['relative_price_ranking_score'].fillna(0, inplace=True) community_summary_df['avg_cost_score'] = 67.0195871700 * ( @@ -54,22 +59,64 @@ def calc_consumption_potential(city): community_summary_df['avg_cost_score'].fillna(0, inplace=True) community_summary_df['consumption_potential'] = community_summary_df['total_price_score'] + community_summary_df[ - 'estate_charge_score'] + community_summary_df['relative_price_ranking_score'] \ - + community_summary_df['avg_cost_score'] + community_summary_df[ - 'avg_cost_score'] + 4099.978248 - pass + 'estate_charge_score'] + community_summary_df['relative_price_ranking_score'] + community_summary_df['avg_cost_score'] + community_summary_df['avg_cost_score'] + 4099.978248 + return community_summary_df -def calc_consumption_rating(): +def gmm_train(total_rate, consumption_df, source, target): + # 高斯混合模型 + gmm = GaussianMixture(n_components=total_rate).fit(consumption_df[[source]]) + consumption_df.loc[:, 'cluster'] = gmm.predict(consumption_df[[source]]) + consumption_df.loc[:, 'price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean') + consumption_df.loc[:, target] = consumption_df['price_mean_cluster'].rank(method='dense').astype(int) + consumption_df = consumption_df.drop(columns=['cluster', 'price_mean_cluster'], axis=1) + return consumption_df + + +def calc_consumption_rating(community_summary_df): """ 计算消费等级 """ - pass + consumption_aim_df = community_summary_df[['_id', 'city', 'consumption_potential']] + consumption_aim_df = consumption_aim_df.dropna(subset=['consumption_potential']) + cities = consumption_aim_df['city'].unique() + source = "consumption_potential" + target = "consumption_rating" + actions = list() + count = 0 + + for city in cities: + df = consumption_aim_df[consumption_aim_df['city'] == city] + if len(df) < 10: + continue + df = gmm_train(10, df, source, target) + # 批量更新 + for index, item in df.iterrows(): + row = dict() + _id = str(item["_id"]) + row['consumption_rating'] = item['consumption_rating'] + row['consumption_potential'] = item['consumption_potential'] + + action = {'_op_type': 'update', # 操作 index update create delete + '_index': index_name, # index + '_id': _id, # 请勿忘记ID + 'doc': row} + actions.append(action) + if len(actions) == 5000: + helpers.bulk(client=es206_client.elastic_client, actions=actions) + count += len(actions) + actions.clear() + log.info("评级中: 当前已评级{}".format(count)) + if len(actions) > 0: + helpers.bulk(client=es206_client.elastic_client, actions=actions) + count += len(actions) + actions.clear() + log.info("评级完成,共更新: {}".format(count)) def sync(city=None): - calc_consumption_potential(city) - pass + community_summary_df = calc_consumption_potential(city) + calc_consumption_rating(community_summary_df) sync() diff --git a/dags/business/business/community/price_rating.py b/dags/business/business/community/price_rating.py new file mode 100644 index 0000000..70d1005 --- /dev/null +++ b/dags/business/business/community/price_rating.py @@ -0,0 +1,84 @@ +""" +楼盘价格评级 +""" +""" +月消费潜力、评级计算 +""" +from goojia_utils import ElasticSearchClient +import pandas as pd +import logging +from elasticsearch import helpers + +from sklearn.mixture import GaussianMixture + +log = logging.getLogger(__name__) + +es206_client = ElasticSearchClient(host='192.168.0.206') +index_name = "community_summary" + +def gmm_train(total_rate, consumption_df, source, target): + # 高斯混合模型 + gmm = GaussianMixture(n_components=total_rate).fit(consumption_df[[source]]) + consumption_df.loc[:, 'cluster'] = gmm.predict(consumption_df[[source]]) + consumption_df.loc[:, 'price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean') + consumption_df.loc[:, target] = consumption_df['price_mean_cluster'].rank(method='dense').astype(int) + consumption_df = consumption_df.drop(columns=['cluster', 'price_mean_cluster'], axis=1) + return consumption_df + + +def calc_price_rating(city=None): + """ + 计算消费等级 + """ + sql = "select id, city, region, name, avg_price from community_summary" + if pd.notna(city): + sql += " where city='" + city + "'" + community_summary_df = es206_client.scan_by_sql_to_data_frame(sql) + + consumption_aim_df = community_summary_df[['_id', 'city', 'avg_price']] + consumption_aim_df = consumption_aim_df.dropna(subset=['avg_price']) + cities = consumption_aim_df['city'].unique() + source = "avg_price" + target = "price_rating" + score_target = "price_score" + actions = list() + count = 0 + + for city in cities: + df = consumption_aim_df[consumption_aim_df['city'] == city] + if len(df) < 10: + continue + df = gmm_train(10, df, source, target) + if len(df) >= 100: + df = gmm_train(100, df, source, score_target) + df['price_score'] /= 10 + + # 批量更新 + for index, item in df.iterrows(): + row = dict() + _id = str(item["_id"]) + row['price_rating'] = item['price_rating'] + row['avg_price'] = item['avg_price'] + + action = {'_op_type': 'update', + '_index': index_name, + '_id': _id, # 请勿忘记ID + 'doc': row} + actions.append(action) + if len(actions) == 5000: + helpers.bulk(client=es206_client.elastic_client, actions=actions) + count += len(actions) + actions.clear() + log.info("评级中: 当前已评级{}".format(count)) + if len(actions) > 0: + helpers.bulk(client=es206_client.elastic_client, actions=actions) + count += len(actions) + actions.clear() + log.info("评级完成,共更新: {}".format(count)) + + +def sync(city=None): + calc_price_rating(city) + + +sync() diff --git a/dags/business/business/rate.py b/dags/business/business/rate.py deleted file mode 100644 index 566880a..0000000 --- a/dags/business/business/rate.py +++ /dev/null @@ -1,25 +0,0 @@ -""" -消费或价格评级定时运行 -""" -import requests -import logging -import sys - -log = logging.getLogger(__name__) - -type = sys.argv[1] if len(sys.argv) > 1 else '' -print("输出评级类型" + type) - -if type == 'price': - log.info("定时楼盘评级开始。。。。。") - headers = {'Content-Type': 'application/json'} - url = "http://192.168.0.19:31080/FangJiaNlpService/price_rate" - if requests.get(url=url, headers=headers).text == 'success': - log.info("定时楼盘评级调用完毕。。。。。") - -else: - log.info("定时消费评级开始。。。。。") - headers = {'Content-Type': 'application/json'} - url = "http://192.168.0.19:31080/FangJiaNlpService/consumption_rate" - if requests.get(url=url, headers=headers).text == 'success': - log.info("定时消费评级调用完毕。。。。。") \ No newline at end of file diff --git a/dags/business/community_summary_dag.py b/dags/business/community_summary_dag.py index 65b53b3..fe09e03 100644 --- a/dags/business/community_summary_dag.py +++ b/dags/business/community_summary_dag.py @@ -51,8 +51,13 @@ with DAG( bash_command='python /opt/airflow/dags/repo/dags/business/business/community/relative_price_ranking.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', ) t11 = BashOperator( - task_id='consumption_potential_dag', - bash_command='python /opt/airflow/dags/repo/dags/business/business/community/consumption_potential.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + task_id='consumption_dag', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/consumption.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + ) + t12 = BashOperator( + task_id='price_rating_dag', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price_rating.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', ) - t1 >> t2 >> t3 >> t4 >> t5 >> t6 >> t7 >> t8 >> t9 >> t10 >> t11 \ No newline at end of file + # t1 >> t2 >> t3 >> t4 >> t5 >> t6 >> t7 >> t8 >> t9 >> t10 >> t11 >> t12 + t11 >> t12 \ No newline at end of file diff --git a/dags/business/rate_dag.py b/dags/business/rate_dag.py deleted file mode 100644 index 1e5223e..0000000 --- a/dags/business/rate_dag.py +++ /dev/null @@ -1,17 +0,0 @@ -from airflow.operators.bash import BashOperator -import pendulum -from airflow import DAG - -with DAG( - "rate_dag", - # 每月3日16点整分运行 - schedule_interval="0 16 3 * *", - start_date=pendulum.datetime(2022, 7, 20, tz="Asia/Shanghai"), - catchup=True, - description="地数价格与消费评级 指定type: price或consumption, 不指定则只对消费评级" -) as dag: - t1 = BashOperator( - task_id='rate_dag', - bash_command='python /opt/airflow/dags/repo/dags/business/business/rate.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - ) - t1 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index cb9761c..56786d1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ goojia_utils nextcloud-api-wrapper -elasticsearch==7.11.0 \ No newline at end of file +elasticsearch==7.11.0 +scikit-learn==0.24.2 \ No newline at end of file