From 48562ba02382250ac2f68e649d2fce741ae4cdc7 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Mon, 21 Nov 2022 10:48:26 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=85=AC=E4=BA=A4=E6=B5=8B?= =?UTF-8?q?=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/business/business/community/traffic.py | 12 +- dags/business/community_summary_dag.py | 138 ++++++++++---------- 2 files changed, 77 insertions(+), 73 deletions(-) diff --git a/dags/business/business/community/traffic.py b/dags/business/business/community/traffic.py index 57954c0..ad8cfb1 100644 --- a/dags/business/business/community/traffic.py +++ b/dags/business/business/community/traffic.py @@ -36,7 +36,9 @@ def deal(actions, item): doc_info['bus_station'] = ",".join(bus_station) bus_doc = poiutils.find_nearest_by_location(curr_city, None, [lng, lat], "交通", "公交") + log.info(bus_doc) if pd.notna(bus_doc): + log.info("开始计算最近公交") doc_info['nearest_bus_station_distance'] = int(bus_doc['sort'][0]) doc_info['nearest_bus_station'] = bus_doc['_source']['name'] @@ -46,10 +48,12 @@ def deal(actions, item): if len(subway_station) > 0: doc_info['subway_station'] = ",".join(subway_station) - bus_doc = poiutils.find_nearest_by_location(curr_city, None, [lng, lat], "交通", "地铁") - if pd.notna(bus_doc): - doc_info['nearest_subway_station_distance'] = int(bus_doc['sort'][0]) - doc_info['nearest_subway_station'] = bus_doc['_source']['name'] + subway_doc = poiutils.find_nearest_by_location(curr_city, None, [lng, lat], "交通", "地铁") + log.info(subway_doc) + if pd.notna(subway_doc): + log.info("开始计算最近地铁") + doc_info['nearest_subway_station_distance'] = int(subway_doc['sort'][0]) + doc_info['nearest_subway_station'] = subway_doc['_source']['name'] action = { "_op_type": "update", diff --git a/dags/business/community_summary_dag.py b/dags/business/community_summary_dag.py index 342d992..4777149 100644 --- a/dags/business/community_summary_dag.py +++ b/dags/business/community_summary_dag.py @@ -10,76 +10,76 @@ with DAG( catchup=True, description="地址输出信息分类数据定时更新" ) as dag: - # t1 = BashOperator( - # task_id='comm', - # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/comm.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - # ) - # t2 = BashOperator( - # task_id='price', - # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - # ) - # t3 = BashOperator( - # task_id='sell', - # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/sell.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - # ) - # t4 = BashOperator( - # task_id='avg_cost', - # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/avg_cost.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - # ) - # t5 = BashOperator( - # task_id='trade', - # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/trade.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - # ) - # t6 = BashOperator( - # task_id='total_price', - # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - # ) - # t6_1 = BashOperator( - # task_id='total_price_complete', - # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price_complete.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - # ) + t1 = BashOperator( + task_id='comm', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/comm.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + ) + t2 = BashOperator( + task_id='price', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + ) + t3 = BashOperator( + task_id='sell', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/sell.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + ) + t4 = BashOperator( + task_id='avg_cost', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/avg_cost.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + ) + t5 = BashOperator( + task_id='trade', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/trade.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + ) + t6 = BashOperator( + task_id='total_price', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + ) + t6_1 = BashOperator( + task_id='total_price_complete', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price_complete.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + ) t7 = BashOperator( task_id='traffic', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/traffic.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', ) - # t7_1 = BashOperator( - # task_id='business_district', - # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/business_district.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - # ) - # t8 = BashOperator( - # task_id='medical', - # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/medical.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - # ) - # t9 = BashOperator( - # task_id='life', - # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/life.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - # ) - # t10 = BashOperator( - # task_id='relative_price_ranking', - # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/relative_price_ranking.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - # ) - # t11 = BashOperator( - # task_id='consumption', - # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/consumption.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - # ) - # t12 = BashOperator( - # task_id='price_rating', - # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price_rating.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - # ) - # t13 = BashOperator( - # task_id='education', - # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - # ) - # t14 = BashOperator( - # task_id='education2', - # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education2.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - # ) - # t15 = BashOperator( - # task_id='extra', - # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/extra.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - # ) - # t16 = BashOperator( - # task_id='other', - # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/other.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - # ) - # t1 >> t2 >> t3 >> t4 >> t5 >> t6 >> t6_1 >> t7 >> t7_1 >> t8 >> t9 >> t10 >> t11 >> t12 >> t13 >> t14 >> t15 >> t16 + t7_1 = BashOperator( + task_id='business_district', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/business_district.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + ) + t8 = BashOperator( + task_id='medical', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/medical.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + ) + t9 = BashOperator( + task_id='life', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/life.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + ) + t10 = BashOperator( + task_id='relative_price_ranking', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/relative_price_ranking.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + ) + t11 = BashOperator( + task_id='consumption', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/consumption.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + ) + t12 = BashOperator( + task_id='price_rating', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price_rating.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + ) + t13 = BashOperator( + task_id='education', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + ) + t14 = BashOperator( + task_id='education2', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/education2.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + ) + t15 = BashOperator( + task_id='extra', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/extra.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + ) + t16 = BashOperator( + task_id='other', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/other.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + ) + t1 >> t2 >> t3 >> t4 >> t5 >> t6 >> t6_1 >> t7 >> t7_1 >> t8 >> t9 >> t10 >> t11 >> t12 >> t13 >> t14 >> t15 >> t16