From 202a5b9198c67ef7c743cd2a71eb243609526f03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Mon, 22 Jul 2024 17:37:38 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/jh/dag_third_gxd.py | 16 ++++++++++++++ dags/jh/price/price_third_gxd.py | 38 +++++++++++++++++++++++++++++--- 2 files changed, 51 insertions(+), 3 deletions(-) create mode 100644 dags/jh/dag_third_gxd.py diff --git a/dags/jh/dag_third_gxd.py b/dags/jh/dag_third_gxd.py new file mode 100644 index 0000000..e3f8493 --- /dev/null +++ b/dags/jh/dag_third_gxd.py @@ -0,0 +1,16 @@ +from airflow.operators.bash import BashOperator +import pendulum +from airflow import DAG + +with DAG( + dag_id="third_gxd_dag", + #schedule_interval="30 16 27 * *", + start_date=pendulum.datetime(2024, 7, 22, tz="Asia/Shanghai"), + catchup=True, + description="第三方均价及国信达价格输出" +) as dag: + t1 = BashOperator( + task_id='gxd_dag', + bash_command='python /opt/airflow/dags/repo/dags/jh/price/price_third_gxd.py {{dag_run.conf["yearmonth"] if "yearmonth" in dag_run.conf}} ', + ) + t1 diff --git a/dags/jh/price/price_third_gxd.py b/dags/jh/price/price_third_gxd.py index a7f5ec5..66d97fb 100644 --- a/dags/jh/price/price_third_gxd.py +++ b/dags/jh/price/price_third_gxd.py @@ -1,9 +1,21 @@ +import logging +import os +import sys + import pandas as pd +import requests +from goojia_utils import dateutils, nextcloudutils + from dags.configs.citymap import city_name_id_map from pqsql_connect import get_community_base_price,get_evaluation_third_price,get_third_price from mongodb_connect import get_district_stat_on,get_seaweed,get_gxd_price -yearmonth='202404' -latest_year_month = '202403' +yearmonth = sys.argv[1] if len(sys.argv) > 1 else dateutils.format_date(pd.datetime.utcnow(), "%Y%m") +latest_year_month = dateutils.format_date(dateutils.add_months(dateutils.parse_date(yearmonth, '%Y%m'), months=-1), '%Y%m') + +pathAll = "/home/airflow/data/third_gxd" +if not os.path.exists(pathAll): + os.makedirs(pathAll) + df_concat = pd.DataFrame() for city in city_name_id_map: print(city) @@ -44,6 +56,26 @@ for city in city_name_id_map: price_merge = price_merge.merge(data_df_temp[['_id','国信达价格']], on='_id', how='left') price_merge.rename(columns={'city': '城市', 'district': '区域', '_id': '小区ID', 'name': '小区名','yearmonth':'月份'}, inplace=True) df_concat = pd.concat([df_concat,price_merge]) -df_concat.to_excel('小区第三方均价输出.xlsx', encoding='utf-8', index=False) +df_concat.to_excel(yearmonth + '小区第三方均价输出.xlsx', encoding='utf-8', index=False) + +output_path = pathAll + "/" + yearmonth + '小区第三方均价输出.xlsx' +df_concat.to_excel(output_path, index=False) + +cloudPath = '共享/统计/' + yearmonth + '小区第三方均价输出.xlsx' +nextcloudutils.up_to_nextcloud(output_path, cloudPath) +logging.info(f"区域涨跌幅数据Excel文件已经上传到云盘") + + +infos = "【第三方价格自动检测】" +data = { + "msgtype": "text", + "text": { + "content": infos + "数据Excel文件已经上传到云盘, 请点击 https://nextcloud.fangjia.com/s/3aw663fy9BbiBeL 查看详情" + } +} + +# 发送消息至群 +requests.post(url='https://oapi.dingtalk.com/robot/send?access_token=1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736', + json=data)