From 36649b28a339c931ea4fa857997a1c0e67ef9199 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Tue, 15 Nov 2022 16:02:35 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E7=89=A9=E4=B8=9A=E8=B4=B9?= =?UTF-8?q?=E7=AD=89=E7=BA=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../community/estate_charge_ranking.py | 88 +++++++++++++++++++ dags/business/community_summary_dag.py | 13 +-- 2 files changed, 96 insertions(+), 5 deletions(-) create mode 100644 dags/business/business/community/estate_charge_ranking.py diff --git a/dags/business/business/community/estate_charge_ranking.py b/dags/business/business/community/estate_charge_ranking.py new file mode 100644 index 0000000..ec693de --- /dev/null +++ b/dags/business/business/community/estate_charge_ranking.py @@ -0,0 +1,88 @@ +""" +物业费等级 +""" +from goojia_utils import ElasticSearchClient, dateutils +import logging +import pandas as pd +from elasticsearch import helpers +import datetime +log = logging.getLogger(__name__) +es206_client = ElasticSearchClient(host='192.168.0.206') + +index_name = "community_summary" +path = "/home/airflow/data/community" +m_date = dateutils.iso_format() + +count = 0 + + +def get_old_or_new(finish_building_date): + """ + 或许新旧 + """ + if pd.isna(finish_building_date): + return "旧" + now_year = datetime.datetime.now().year + if now_year - int(finish_building_date) < 3: + return "新" + return "旧" + + +def deal(actions, x): + global count + estate_charge_ranking = '' + if pd.notna(x['estate_charge']): + estate_charge = float(x['estate_charge']) + if estate_charge >= 1: + estate_charge_ranking = "一级" + if estate_charge >= 0.75: + estate_charge_ranking = "二级" + if estate_charge >= 0.50: + estate_charge_ranking = "三级" + if estate_charge >= 0.35: + estate_charge_ranking = "四级" + if estate_charge < 0.35: + estate_charge_ranking = "五级" + else: + old_or_new = get_old_or_new(x['finish_building_date']) + if old_or_new == '新': + if pd.notna(x['build_type']) and '高层' in x['build_type']: + estate_charge_ranking = '三级' + else: + estate_charge_ranking = '四级' + elif old_or_new == '旧': + estate_charge_ranking = '五级' + + action_doc = { + "_op_type": "update", + "_index": index_name, + "_id": x['_id'], + "doc": {'estate_charge_ranking': estate_charge_ranking} + } + actions.append(action_doc) + if len(actions) == 5000: + helpers.bulk(es206_client.elastic_client, actions) + count += len(actions) + log.info(f"物业费等级已更新{count}") + actions.clear() + + +def sync(city=None): + log.info(f"开始同步物业费等级至206 {index_name}") + + sql = "select city, region, name, estate_charge, finish_building_date, build_type from community_summary " + if pd.notna(city): + sql += " where city='" + city + "'" + community_summary_df = es206_client.scan_by_sql_to_data_frame(sql) + + actions = list() + community_summary_df.apply(lambda x: deal(actions, x), axis=1) + + if len(actions) > 0: + helpers.bulk(es206_client.elastic_client, actions) + actions.clear() + + log.info(f"完成同步物业费等级至206 {index_name}") + + +sync() \ No newline at end of file diff --git a/dags/business/community_summary_dag.py b/dags/business/community_summary_dag.py index 5a42367..83d5c34 100644 --- a/dags/business/community_summary_dag.py +++ b/dags/business/community_summary_dag.py @@ -70,9 +70,12 @@ with DAG( # task_id='education2_dag', # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education2.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', # ) - t15 = BashOperator( - task_id='extra_dag', - bash_command='python /opt/airflow/dags/repo/dags/business/business/community/extra.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + # t15 = BashOperator( + # task_id='extra_dag', + # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/extra.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + # ) + t16 = BashOperator( + task_id='estate_charge_ranking_dag', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/estate_charge_ranking.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', ) - t15 - # t1 >> t2 >> t3 >> t4 >> t5 >> t6 >> t6_1 >> t7 >> t8 >> t9 >> t10 >> t11 >> t12 >> t13 >> t14 >> t15 + # t1 >> t2 >> t3 >> t4 >> t5 >> t6 >> t6_1 >> t7 >> t8 >> t9 >> t10 >> t11 >> t12 >> t13 >> t14 >> t15 >> t16