From 0281a2d5f27d318764a9c8a60b4a7ca33d804fba Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Mon, 21 Nov 2022 17:39:30 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E7=89=A9=E4=B8=9A=E8=B4=B9?= =?UTF-8?q?=E7=AD=89=E7=BA=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../community/estate_charge_ranking.py | 79 +++++++++++++++++++ dags/business/business/community/other.py | 45 +---------- dags/business/community_summary_dag.py | 6 +- 3 files changed, 88 insertions(+), 42 deletions(-) create mode 100644 dags/business/business/community/estate_charge_ranking.py diff --git a/dags/business/business/community/estate_charge_ranking.py b/dags/business/business/community/estate_charge_ranking.py new file mode 100644 index 0000000..cf7adf3 --- /dev/null +++ b/dags/business/business/community/estate_charge_ranking.py @@ -0,0 +1,79 @@ +""" +物业费等级计算 +""" +from goojia_utils import ElasticSearchClient +import pandas as pd +import numpy as np +import logging +from elasticsearch import helpers + +from sklearn.mixture import GaussianMixture + +log = logging.getLogger(__name__) + +es206_client = ElasticSearchClient(host='192.168.0.206') +index_name = "community_summary" +path = "/opt/airflow/dags/repo/dags/business/business/community/data" + + + +def gmm_train(total_rate, consumption_df, source, target): + # 高斯混合模型 + gmm = GaussianMixture(n_components=total_rate).fit(consumption_df[[source]]) + consumption_df.loc[:, 'cluster'] = gmm.predict(consumption_df[[source]]) + consumption_df.loc[:, 'price_mean_cluster'] = consumption_df.groupby('cluster')[source].transform('mean') + consumption_df.loc[:, target] = consumption_df['price_mean_cluster'].rank(method='dense', ascending=False).astype(int) + consumption_df = consumption_df.drop(columns=['cluster', 'price_mean_cluster'], axis=1) + return consumption_df + + +def calc_estate_charge_rating(community_summary_df): + """ + 计算物业费等级 + """ + consumption_aim_df = community_summary_df[['_id', 'city', 'estate_charge']] + consumption_aim_df = consumption_aim_df.dropna(subset=['estate_charge']) + cities = consumption_aim_df['city'].unique() + source = "estate_charge" + target = "estate_charge_ranking" + actions = list() + count = 0 + + for city in cities: + df = consumption_aim_df[consumption_aim_df['city'] == city].copy() + if len(df) < 5: + continue + df = gmm_train(5, df, source, target) + # 批量更新 + for index, item in df.iterrows(): + row = dict() + _id = str(item["_id"]) + row['estate_charge_ranking'] = item['estate_charge_ranking'] + + action = {'_op_type': 'update', # 操作 index update create delete + '_index': index_name, # index + '_id': _id, # 请勿忘记ID + 'doc': row, + "_retry_on_conflict": 5 + } + actions.append(action) + if len(actions) == 5000: + helpers.bulk(client=es206_client.elastic_client, actions=actions) + count += len(actions) + actions.clear() + log.info("评级中: 当前已评级{}".format(count)) + if len(actions) > 0: + helpers.bulk(client=es206_client.elastic_client, actions=actions) + count += len(actions) + actions.clear() + log.info("评级完成,共更新: {}".format(count)) + + +def sync(city=None): + sql = "select id, city, region, name, estate_charge from community_summary" + if pd.notna(city): + sql += " where city='" + city + "'" + community_summary_df = es206_client.scan_by_sql_to_data_frame(sql) + calc_estate_charge_rating(community_summary_df) + +sync() diff --git a/dags/business/business/community/other.py b/dags/business/business/community/other.py index 35351b1..0f02d66 100644 --- a/dags/business/business/community/other.py +++ b/dags/business/business/community/other.py @@ -16,46 +16,9 @@ m_date = dateutils.iso_format() count = 0 -def get_old_or_new(finish_building_date): - """ - 或许新旧 - """ - if pd.isna(finish_building_date): - return "旧" - now_year = datetime.datetime.now().year - if now_year - int(finish_building_date) < 3: - return "新" - return "旧" - - def deal(actions, x): global count - estate_charge_ranking = None - if pd.notna(x['estate_charge']): - estate_charge = float(x['estate_charge']) - if estate_charge >= 1: - estate_charge_ranking = "一级" - elif estate_charge >= 0.75: - estate_charge_ranking = "二级" - elif estate_charge >= 0.50: - estate_charge_ranking = "三级" - elif estate_charge >= 0.35: - estate_charge_ranking = "四级" - elif estate_charge < 0.35: - estate_charge_ranking = "五级" - else: - old_or_new = get_old_or_new(x['finish_building_date']) - if old_or_new == '新': - if pd.notna(x['build_type']) and '高层' in x['build_type']: - estate_charge_ranking = '三级' - else: - estate_charge_ranking = '四级' - elif old_or_new == '旧': - estate_charge_ranking = '五级' - doc_info = dict() - if pd.notna(estate_charge_ranking): - doc_info['estate_charge_ranking'] = estate_charge_ranking if pd.notna(x['avg_price']) and pd.notna(x['rent_price']): doc_info['sell_to_rent'] = int(x['avg_price'] / x['rent_price']) if pd.notna(x['household_count']): @@ -72,14 +35,14 @@ def deal(actions, x): if len(actions) == 5000: helpers.bulk(es206_client.elastic_client, actions) count += len(actions) - log.info(f"物业费等级、售租比、人口数已更新{count}") + log.info(f"售租比、人口数已更新{count}") actions.clear() def sync(city=None): - log.info(f"开始同步物业费等级、售租比、人口数至206 {index_name}") + log.info(f"开始同步售租比、人口数至206 {index_name}") - sql = "select city, region, name, estate_charge, finish_building_date, build_type, avg_price, rent_price, household_count from community_summary" + sql = "select city, region, name, finish_building_date, build_type, avg_price, rent_price, household_count from community_summary" if pd.notna(city): sql += " where city='" + city + "'" community_summary_df = es206_client.scan_by_sql_to_data_frame(sql) @@ -91,7 +54,7 @@ def sync(city=None): helpers.bulk(es206_client.elastic_client, actions) actions.clear() - log.info(f"完成同步物业费等级、售租比、人口数至206 {index_name}") + log.info(f"完成同步售租比、人口数至206 {index_name}") sync() \ No newline at end of file diff --git a/dags/business/community_summary_dag.py b/dags/business/community_summary_dag.py index 4777149..f100526 100644 --- a/dags/business/community_summary_dag.py +++ b/dags/business/community_summary_dag.py @@ -62,6 +62,10 @@ with DAG( task_id='consumption', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/consumption.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', ) + t11_1 = BashOperator( + task_id='estate_charge_ranking', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/estate_charge_ranking.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + ) t12 = BashOperator( task_id='price_rating', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price_rating.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', @@ -82,4 +86,4 @@ with DAG( task_id='other', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/other.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', ) - t1 >> t2 >> t3 >> t4 >> t5 >> t6 >> t6_1 >> t7 >> t7_1 >> t8 >> t9 >> t10 >> t11 >> t12 >> t13 >> t14 >> t15 >> t16 + t1 >> t2 >> t3 >> t4 >> t5 >> t6 >> t6_1 >> t7 >> t7_1 >> t8 >> t9 >> t10 >> t11 >> t11_1 >> t12 >> t13 >> t14 >> t15 >> t16