From 40aad80afe14f408b15debf9fbe3810e1c99953e Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Tue, 9 May 2023 11:05:50 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=A0=BC=E5=AD=90=E5=9D=87?= =?UTF-8?q?=E4=BB=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/business/business/grid_price.py | 71 ++++++++++++++++++++++++++++ dags/business/dag_grid_price.py | 17 +++++++ 2 files changed, 88 insertions(+) create mode 100644 dags/business/business/grid_price.py create mode 100644 dags/business/dag_grid_price.py diff --git a/dags/business/business/grid_price.py b/dags/business/business/grid_price.py new file mode 100644 index 0000000..88fa42d --- /dev/null +++ b/dags/business/business/grid_price.py @@ -0,0 +1,71 @@ +import logging +import sys +from goojia_utils import ElasticSearchClient, commutils +import pandas as pd +import time + +log = logging.getLogger(__name__) + +city_name = sys.argv[1] if len(sys.argv) > 1 else None +if pd.notna(city_name): + log.info("同步城市:" + city_name) + +es206_client = ElasticSearchClient(host='192.168.0.206') + + +def find_comm_by_grid(city, geo_shape): + comm_info_df = es206_client.scan_to_data_frame('community_summary', { + "_source": ["id", "city", "region", "name", "estate_type"], + "query": { + "bool": { + "filter": [ + { + "term": { + "city": { + "value": city + } + } + }, + { + "geo_polygon": { + "location": { + "points": geo_shape + } + } + } + ] + } + } + }) + return comm_info_df + +estate_type_list = ["普通住宅", "别墅", "商住"] + +def sync(city): + sql = f"select * from grid " + if pd.notna(city): + sql += f" where city='{city}' " + origin_grid_df = es206_client.scan_by_sql_to_data_frame(sql) + cities = origin_grid_df['city'].unique() + + for curr_city in cities: + grid_df = origin_grid_df[origin_grid_df['city'] == curr_city].copy() + avg_price_df = commutils.get_avg_price(curr_city) + avg_price_df = avg_price_df[['city', 'region', 'district_name', 'avg_price']] + avg_price_df = avg_price_df.rename(columns={'district_name': 'name'}) + # 求解格子均价 + actions = list() + for index, item in grid_df.iterrows(): + comm_info_df = find_comm_by_grid(item['city'], item['fence']['coordinates'][0]) + action = dict() + ret = dict() + for estate_type in estate_type_list: + # 计算不同物业类型的小区均价 + comm_info_estate_type_df = comm_info_df[comm_info_df['estate_type'] == estate_type].copy() + if len(comm_info_estate_type_df) > 0: + comm_info_estate_type_df = pd.merge(comm_info_estate_type_df, avg_price_df, how='left', on=['city', 'region', 'name']) + comm_info_estate_type_df = comm_info_estate_type_df[comm_info_estate_type_df['avg_price'].notna()] + if len(comm_info_estate_type_df) > 0: + ret[estate_type] = int(comm_info_estate_type_df['avg_price'].mean()) + action['district_price'] = {''} +sync(city_name) diff --git a/dags/business/dag_grid_price.py b/dags/business/dag_grid_price.py new file mode 100644 index 0000000..3eab1f1 --- /dev/null +++ b/dags/business/dag_grid_price.py @@ -0,0 +1,17 @@ +from airflow.operators.bash import BashOperator +import pendulum +from airflow import DAG + +with DAG( + dag_id="grid_price_dag", + # 每周二运行 + schedule_interval="0 3 * * 2", + start_date=pendulum.datetime(2023, 5, 9, tz="Asia/Shanghai"), + catchup=True, + description="全国格子价格计算" +) as dag: + t1 = BashOperator( + task_id='grid_price', + bash_command='python /opt/airflow/dags/repo/dags/business/business/grid_price.py {{dag_run.conf["city"] if "city" in dag_run.conf}}', + ) + t1 \ No newline at end of file