版本修改
This commit is contained in:
+23
-23
@@ -1,23 +1,23 @@
|
||||
from airflow.operators.bash import BashOperator
|
||||
import pendulum
|
||||
from airflow import DAG
|
||||
|
||||
with DAG(
|
||||
dag_id="ai_stock_dag",
|
||||
# 每天运行
|
||||
schedule="30 8 * * 1-5",
|
||||
start_date=pendulum.datetime(2025, 11, 10, tz="Asia/Shanghai"),
|
||||
catchup=True,
|
||||
tags=["ai_stock"],
|
||||
) as dag:
|
||||
|
||||
t1 = BashOperator(
|
||||
task_id='stock_strategy',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/ai/stock/stock_strategy.py',
|
||||
)
|
||||
|
||||
t2 = BashOperator(
|
||||
task_id='ai_stock',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/ai/stock/stock.py',
|
||||
)
|
||||
t1 >> t2
|
||||
# from airflow.operators.bash import BashOperator
|
||||
# import pendulum
|
||||
# from airflow import DAG
|
||||
#
|
||||
# with DAG(
|
||||
# dag_id="ai_stock_dag",
|
||||
# # 每天运行
|
||||
# schedule="30 8 * * 1-5",
|
||||
# start_date=pendulum.datetime(2025, 11, 10, tz="Asia/Shanghai"),
|
||||
# catchup=True,
|
||||
# tags=["ai_stock"],
|
||||
# ) as dag:
|
||||
#
|
||||
# t1 = BashOperator(
|
||||
# task_id='stock_strategy',
|
||||
# bash_command='python /opt/airflow/dags/repo/dags/ai/stock/stock_strategy.py',
|
||||
# )
|
||||
#
|
||||
# t2 = BashOperator(
|
||||
# task_id='ai_stock',
|
||||
# bash_command='python /opt/airflow/dags/repo/dags/ai/stock/stock.py',
|
||||
# )
|
||||
# t1 >> t2
|
||||
+38
-38
@@ -1,38 +1,38 @@
|
||||
from goojia_utils import MysqlClient, ArkAIClient, dingutils
|
||||
from goojia_utils import dateutils
|
||||
from datetime import datetime
|
||||
import requests
|
||||
|
||||
db_util = MysqlClient(
|
||||
host="192.168.0.93",
|
||||
port=3306,
|
||||
database="stock",
|
||||
user="root",
|
||||
password="Goojia123456"
|
||||
)
|
||||
date = int(dateutils.format_date(dateutils.add_months(datetime.today(), months=-1), fm='%Y%m%d'))
|
||||
select_stocks = db_util.scan_by_sql_to_data_frame("select ts_code as 股票代码, stock_name as 股票名称,close as 股价, pe as PE,turnover_rate as 换手率 from mu_stocks_new where `trade_date`= (select max(`trade_date`) from mu_stocks_new)")
|
||||
stat = db_util.scan_by_sql_to_data_frame(f"select ts_code as 股票代码, count(ts_code) as 命中次数 from mu_stocks_new where `trade_date` > {date} group by ts_code")
|
||||
select_stocks = select_stocks.merge(stat, on='股票代码', how='left')
|
||||
select_stocks['命中次数'] = select_stocks['命中次数'].fillna(0)
|
||||
select_stocks = select_stocks.sort_values(by=['命中次数', 'PE'], ascending=[False, True])
|
||||
def df_to_nested_list(df):
|
||||
"""转换为嵌套的 Markdown 列表"""
|
||||
lines = []
|
||||
i= 1
|
||||
for _, row in df.iterrows():
|
||||
lines.append(f"{i}. **{row['股票名称']}** ({row['股票代码']}) PE: {row['PE']} 股价: {row['股价']:.2f} 出现次数: {row['命中次数']} 换手率: {row['换手率']:.2f}")
|
||||
i = i + 1
|
||||
return '\n'.join(lines)
|
||||
data = {
|
||||
"msgtype": "markdown",
|
||||
"markdown": {
|
||||
"title": dateutils.format_date(datetime.today(), fm='%Y%m%d') + "选股策略播报",
|
||||
"text": df_to_nested_list(select_stocks)
|
||||
}
|
||||
}
|
||||
access_token = "1e5f9aaeb90b091bc79652b52800cad2f1dee1ac0990149252511965f6d1c12a"
|
||||
# 发送消息至群
|
||||
msg = requests.post(url='https://oapi.dingtalk.com/robot/send?access_token=' + access_token, json=data).json()
|
||||
msg = requests.post(url='https://oapi.dingtalk.com/robot/send?access_token=45ac47a34dc5491535d77d643512244a85d91d7e74676e9c42446dc06357326e', json=data).json()
|
||||
print(msg)
|
||||
# from goojia_utils import MysqlClient, ArkAIClient, dingutils
|
||||
# from goojia_utils import dateutils
|
||||
# from datetime import datetime
|
||||
# import requests
|
||||
#
|
||||
# db_util = MysqlClient(
|
||||
# host="192.168.0.93",
|
||||
# port=3306,
|
||||
# database="stock",
|
||||
# user="root",
|
||||
# password="Goojia123456"
|
||||
# )
|
||||
# date = int(dateutils.format_date(dateutils.add_months(datetime.today(), months=-1), fm='%Y%m%d'))
|
||||
# select_stocks = db_util.scan_by_sql_to_data_frame("select ts_code as 股票代码, stock_name as 股票名称,close as 股价, pe as PE,turnover_rate as 换手率 from mu_stocks_new where `trade_date`= (select max(`trade_date`) from mu_stocks_new)")
|
||||
# stat = db_util.scan_by_sql_to_data_frame(f"select ts_code as 股票代码, count(ts_code) as 命中次数 from mu_stocks_new where `trade_date` > {date} group by ts_code")
|
||||
# select_stocks = select_stocks.merge(stat, on='股票代码', how='left')
|
||||
# select_stocks['命中次数'] = select_stocks['命中次数'].fillna(0)
|
||||
# select_stocks = select_stocks.sort_values(by=['命中次数', 'PE'], ascending=[False, True])
|
||||
# def df_to_nested_list(df):
|
||||
# """转换为嵌套的 Markdown 列表"""
|
||||
# lines = []
|
||||
# i= 1
|
||||
# for _, row in df.iterrows():
|
||||
# lines.append(f"{i}. **{row['股票名称']}** ({row['股票代码']}) PE: {row['PE']} 股价: {row['股价']:.2f} 出现次数: {row['命中次数']} 换手率: {row['换手率']:.2f}")
|
||||
# i = i + 1
|
||||
# return '\n'.join(lines)
|
||||
# data = {
|
||||
# "msgtype": "markdown",
|
||||
# "markdown": {
|
||||
# "title": dateutils.format_date(datetime.today(), fm='%Y%m%d') + "选股策略播报",
|
||||
# "text": df_to_nested_list(select_stocks)
|
||||
# }
|
||||
# }
|
||||
# access_token = "1e5f9aaeb90b091bc79652b52800cad2f1dee1ac0990149252511965f6d1c12a"
|
||||
# # 发送消息至群
|
||||
# msg = requests.post(url='https://oapi.dingtalk.com/robot/send?access_token=' + access_token, json=data).json()
|
||||
# msg = requests.post(url='https://oapi.dingtalk.com/robot/send?access_token=45ac47a34dc5491535d77d643512244a85d91d7e74676e9c42446dc06357326e', json=data).json()
|
||||
# print(msg)
|
||||
@@ -1,59 +1,59 @@
|
||||
from airflow.providers.docker.operators.docker import DockerOperator
|
||||
from airflow.providers.cncf.kubernetes.operators.kubernetes_pod import KubernetesPodOperator
|
||||
import pendulum
|
||||
from airflow import DAG
|
||||
from docker.types import Mount
|
||||
from kubernetes.client import models as k8s
|
||||
|
||||
with DAG(
|
||||
dag_id="comm_add_dag",
|
||||
# 每月20日1点整分运行
|
||||
schedule="0 1 20 * *",
|
||||
start_date=pendulum.datetime(2023, 5, 8, tz="Asia/Shanghai"),
|
||||
catchup=True,
|
||||
description="小区自动化入库"
|
||||
) as dag:
|
||||
# k = KubernetesPodOperator(
|
||||
# name="comm-add",
|
||||
# from airflow.providers.docker.operators.docker import DockerOperator
|
||||
# from airflow.providers.cncf.kubernetes.operators.kubernetes_pod import KubernetesPodOperator
|
||||
# import pendulum
|
||||
# from airflow import DAG
|
||||
# from docker.types import Mount
|
||||
# from kubernetes.client import models as k8s
|
||||
#
|
||||
# with DAG(
|
||||
# dag_id="comm_add_dag",
|
||||
# # 每月20日1点整分运行
|
||||
# schedule="0 1 20 * *",
|
||||
# start_date=pendulum.datetime(2023, 5, 8, tz="Asia/Shanghai"),
|
||||
# catchup=True,
|
||||
# description="小区自动化入库"
|
||||
# ) as dag:
|
||||
# # k = KubernetesPodOperator(
|
||||
# # name="comm-add",
|
||||
# # image="192.168.0.19:9012/comm-add:1.0.1",
|
||||
# # cmds=["bash", "-cx"],
|
||||
# # arguments=["echo", "10"],
|
||||
# # labels={"foo": "bar"},
|
||||
# # task_id="comm_add",
|
||||
# # do_xcom_push=True,
|
||||
# # )
|
||||
# # k.dry_run()
|
||||
#
|
||||
# production_task = KubernetesPodOperator(namespace='airflow-plus',
|
||||
# # image="my-production-job:release-1.0.1", <-- old release
|
||||
# image="192.168.0.19:9012/comm-add:1.0.1",
|
||||
# cmds=["bash", "-cx"],
|
||||
# arguments=["echo", "10"],
|
||||
# labels={"foo": "bar"},
|
||||
# image_pull_policy="Always",
|
||||
# image_pull_secrets=[k8s.V1LocalObjectReference("nexus-registry-secret")],
|
||||
# cmds=["python", "main.py"],
|
||||
# arguments=["--city", '{{dag_run.conf["city"] if "city" in dag_run.conf else "all"}}'],
|
||||
# name="comm_add",
|
||||
# task_id="comm_add",
|
||||
# do_xcom_push=True,
|
||||
# get_logs=True
|
||||
# )
|
||||
# k.dry_run()
|
||||
|
||||
production_task = KubernetesPodOperator(namespace='airflow-plus',
|
||||
# image="my-production-job:release-1.0.1", <-- old release
|
||||
image="192.168.0.19:9012/comm-add:1.0.1",
|
||||
image_pull_policy="Always",
|
||||
image_pull_secrets=[k8s.V1LocalObjectReference("nexus-registry-secret")],
|
||||
cmds=["python", "main.py"],
|
||||
arguments=["--city", '{{dag_run.conf["city"] if "city" in dag_run.conf else "all"}}'],
|
||||
name="comm_add",
|
||||
task_id="comm_add",
|
||||
get_logs=True
|
||||
)
|
||||
|
||||
production_task
|
||||
# t1 = DockerOperator(
|
||||
# # 刚才定义的conn id
|
||||
# docker_conn_id="docker_repo",
|
||||
# # WebServer中点击Admin -> Providers -> Docker对应的Version,如果是auto,可能会报错
|
||||
# api_version="2.2.0",
|
||||
# auto_remove=True,
|
||||
# docker_url='unix://var/run/docker.sock',
|
||||
# network_mode='bridge',
|
||||
# image='192.168.0.19:9012/comm-add:1.0.1',
|
||||
# # 临沂为我们的传参,python代码中, 镜像中部分代码
|
||||
# # @ click.command()
|
||||
# # @ click.option('--city', default="all", help='要计算的城市')
|
||||
# #command='python main.py --city {{dag_run.conf["city"] if "city" in dag_run.conf}} else all',
|
||||
# command='echo "this is a test message shown from within the container',
|
||||
# mounts=[
|
||||
# Mount(source="/var/run/docker.sock", target="/var/run/docker.sock", type="bind")
|
||||
# ],
|
||||
# task_id='comm_add'
|
||||
# )
|
||||
# t1
|
||||
#
|
||||
# production_task
|
||||
# # t1 = DockerOperator(
|
||||
# # # 刚才定义的conn id
|
||||
# # docker_conn_id="docker_repo",
|
||||
# # # WebServer中点击Admin -> Providers -> Docker对应的Version,如果是auto,可能会报错
|
||||
# # api_version="2.2.0",
|
||||
# # auto_remove=True,
|
||||
# # docker_url='unix://var/run/docker.sock',
|
||||
# # network_mode='bridge',
|
||||
# # image='192.168.0.19:9012/comm-add:1.0.1',
|
||||
# # # 临沂为我们的传参,python代码中, 镜像中部分代码
|
||||
# # # @ click.command()
|
||||
# # # @ click.option('--city', default="all", help='要计算的城市')
|
||||
# # #command='python main.py --city {{dag_run.conf["city"] if "city" in dag_run.conf}} else all',
|
||||
# # command='echo "this is a test message shown from within the container',
|
||||
# # mounts=[
|
||||
# # Mount(source="/var/run/docker.sock", target="/var/run/docker.sock", type="bind")
|
||||
# # ],
|
||||
# # task_id='comm_add'
|
||||
# # )
|
||||
# # t1
|
||||
@@ -3,7 +3,7 @@
|
||||
"""
|
||||
from airflow.operators.bash import BashOperator
|
||||
from airflow.operators.python import PythonOperator, BranchPythonOperator
|
||||
from office.rent.base_price import calc
|
||||
from rent.base_price import calc
|
||||
import pendulum
|
||||
from airflow import DAG
|
||||
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
from airflow.operators.bash import BashOperator
|
||||
from airflow.operators.python import PythonOperator, BranchPythonOperator
|
||||
from airflow.operators.empty import EmptyOperator
|
||||
from office.sell.base_price import calc
|
||||
from office.sell.fill_base_price import fill_base_price
|
||||
from sell.base_price import calc
|
||||
from sell.fill_base_price import fill_base_price
|
||||
import pendulum
|
||||
|
||||
from airflow import DAG
|
||||
|
||||
@@ -1,249 +1,249 @@
|
||||
"""
|
||||
统计城市 区域 价格涨跌幅
|
||||
"""
|
||||
import logging
|
||||
import pandas as pd
|
||||
from sqlalchemy import create_engine
|
||||
from goojia_utils import dateutils
|
||||
import datetime
|
||||
import os
|
||||
pathAll = "/home/airflow/data/cityPrice"
|
||||
if not os.path.exists(pathAll):
|
||||
os.makedirs(pathAll)
|
||||
# 创建数据库引擎
|
||||
engine235 = create_engine("postgresql+psycopg2://postgres:Goojia123456@192.168.0.235:5433/fangjia?client_encoding=utf8",
|
||||
echo=False)
|
||||
pd.set_option('mode.chained_assignment', None)
|
||||
year_month_day = dateutils.format_date(datetime.datetime.now())
|
||||
# 执行SQL查询获取城市价格数据
|
||||
city_sql = "SELECT DISTINCT urbn_nm FROM price_stat.avm_city_price"
|
||||
city_names_set = set(pd.read_sql(city_sql, engine235)['urbn_nm'])
|
||||
city_names = list(city_names_set)
|
||||
# 假设这是你的pgsql数据
|
||||
all_city_data = pd.DataFrame()
|
||||
# 执行SQL查询获取区域价格数据
|
||||
region_sql = "SELECT DISTINCT urbn_nm,gxda_admndiv_nm FROM price_stat.avm_district_price"
|
||||
region_data = pd.read_sql(region_sql, engine235)
|
||||
city_data_useinregion = pd.DataFrame()
|
||||
# 假设这是你的pgsql数据
|
||||
all_region_data = pd.DataFrame()
|
||||
|
||||
def up_to_nextcloud(aim_file_name,aim_file_path):
|
||||
"""
|
||||
上传统计信息至nextcloud
|
||||
"""
|
||||
from nextcloud import NextCloud
|
||||
NEXTCLOUD_URL = "https://nextcloud.fangjia.com"
|
||||
NEXTCLOUD_USERNAME = "shenjian@fangjia.com"
|
||||
NEXTCLOUD_PASSWORD = "goojia123456"
|
||||
nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD, json_output=True)
|
||||
nxc.upload_file(aim_file_name, aim_file_path)
|
||||
def city_increase(city_names, all_city_data):
|
||||
global city_data_useinregion # 声明为全局变量
|
||||
# 遍历城市列表,对每个城市进行查询
|
||||
for city in city_names:
|
||||
query = f"SELECT urbn_nm as 城市, gujiabds_clc_yrmo_val as 月份, gujia_bds_clc_avprc as 估家计算均价, gujia_bds_urbn_avprc as 估家城市均价 FROM price_stat.avm_city_price WHERE urbn_nm = '{city}'"
|
||||
city_data = pd.read_sql(query, engine235)
|
||||
|
||||
if city_data.empty:
|
||||
continue
|
||||
|
||||
# 复制一份原始的日期列,用于计算涨跌幅时保持原始数据不变
|
||||
city_data['original_date'] = city_data['月份']
|
||||
|
||||
# 将日期列转换为日期类型
|
||||
city_data['月份'] = pd.to_datetime(city_data['月份'], format='%Y%m')
|
||||
|
||||
# 对价格数据按照月份排序
|
||||
city_data.sort_values(by='月份', inplace=True)
|
||||
|
||||
# 计算涨跌幅
|
||||
city_data['涨跌幅'] = city_data['估家计算均价'].diff() / city_data['估家计算均价'].shift(1) * 100
|
||||
|
||||
# 计算涨跌幅,并保留两位小数,展示为百分比形式
|
||||
price_change = city_data['估家计算均价'].pct_change() * 100
|
||||
city_data['涨跌幅'] = (price_change[price_change != 0].fillna(0).round(2).astype(str) + '%').replace('0.0%', '')
|
||||
|
||||
# 将原始日期列替换为计算涨跌幅后的日期列
|
||||
city_data['月份'] = city_data['original_date']
|
||||
|
||||
# 删除临时列
|
||||
city_data.drop(columns=['original_date'], inplace=True)
|
||||
|
||||
# 将城市数据添加到总的数据中
|
||||
all_city_data = pd.concat([all_city_data, city_data], ignore_index=True)
|
||||
city_data_useinregion = all_city_data
|
||||
return all_city_data
|
||||
|
||||
|
||||
def city_missing():
|
||||
df = city_increase(city_names, all_city_data)
|
||||
df['月份1'] = df['月份']
|
||||
# 将日期字符串转换为日期时间对象
|
||||
df["月份"] = pd.to_datetime(df["月份"], format="%Y%m")
|
||||
|
||||
# 获取当前时间
|
||||
current_time = pd.Timestamp.today().replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
|
||||
# 计算前一年到上月的时间范围
|
||||
one_year_ago = current_time - pd.DateOffset(years=1, days=current_time.day - 1)
|
||||
last_month = current_time - pd.DateOffset(months=1, days=current_time.day - 1)
|
||||
|
||||
# 创建一个包含所有可能月份的日期范围
|
||||
all_months = pd.date_range(start=one_year_ago, end=last_month, freq="MS")
|
||||
|
||||
# 获取每个城市缺失的月份
|
||||
missing_months = []
|
||||
for city, group in df.groupby("城市"):
|
||||
available_months = group["月份"]
|
||||
missing = all_months.difference(available_months)
|
||||
if len(missing) > 0:
|
||||
missing_str = ",".join([m.strftime("%Y%m") for m in missing])
|
||||
missing_months.append({"城市": city, "缺失月份": missing_str})
|
||||
if len(missing_months) > 0:
|
||||
# 创建DataFrame来存储结果
|
||||
missing_df = pd.DataFrame(missing_months)
|
||||
path = pathAll+'/城市价格数据缺失'+year_month_day+'.xlsx'
|
||||
missing_df.to_excel(path, index=False)
|
||||
logging.info(f"城市价格缺失数据已写入Excel文件:{path}")
|
||||
|
||||
cloudPath = '共享/统计/城市区域涨跌幅/' + '城市价格数据缺失'+year_month_day+'.xlsx'
|
||||
up_to_nextcloud(path, cloudPath)
|
||||
logging.info(f"区域涨跌幅数据Excel文件已经上传到云盘")
|
||||
|
||||
# print(missing_df)
|
||||
df['月份'] = df['月份1']
|
||||
# 删除月份1列
|
||||
df.drop(columns=['月份1'], inplace=True)
|
||||
output_path = pathAll+'/城市价格涨跌幅'+year_month_day+'.xlsx'
|
||||
df.to_excel(output_path, index=False)
|
||||
logging.info(f"城市涨跌幅数据已写入Excel文件:{output_path}")
|
||||
|
||||
cloudPath = '共享/统计/城市区域涨跌幅/' + '城市价格涨跌幅'+year_month_day+'.xlsx'
|
||||
up_to_nextcloud(output_path, cloudPath)
|
||||
logging.info(f"区域涨跌幅数据Excel文件已经上传到云盘")
|
||||
|
||||
|
||||
def region_increase(region_data, all_region_data):
|
||||
# 遍历区域列表,对每个区域进行查询
|
||||
for index, row in region_data.iterrows():
|
||||
city = row['urbn_nm']
|
||||
region = row['gxda_admndiv_nm']
|
||||
query = f"SELECT urbn_nm as 城市, gxda_admndiv_nm as 区域, gujiabds_clc_yrmo_val as 月份, gujia_bds_clc_avprc as 估家计算均价, gujia_bds_urbn_avprc as 估家城市均价 FROM price_stat.avm_district_price WHERE urbn_nm = '{city}' AND gxda_admndiv_nm = '{region}'"
|
||||
region_data = pd.read_sql(query, engine235)
|
||||
|
||||
if region_data.empty:
|
||||
continue
|
||||
|
||||
# 复制一份原始的日期列,用于计算涨跌幅时保持原始数据不变
|
||||
region_data['original_date'] = region_data['月份']
|
||||
|
||||
# 将日期列转换为日期类型
|
||||
region_data['月份'] = pd.to_datetime(region_data['月份'], format='%Y%m')
|
||||
|
||||
# 对价格数据按照月份排序
|
||||
region_data.sort_values(by='月份', inplace=True)
|
||||
|
||||
# 计算涨跌幅
|
||||
region_data['涨跌幅'] = region_data['估家计算均价'].diff() / region_data['估家计算均价'].shift(1) * 100
|
||||
|
||||
# 计算涨跌幅,并保留两位小数,展示为百分比形式
|
||||
price_change = region_data['估家计算均价'].pct_change() * 100
|
||||
region_data['涨跌幅'] = (price_change[price_change != 0].fillna(0).round(2).astype(str) + '%').replace('0.0%',
|
||||
'')
|
||||
|
||||
# 将原始日期列替换为计算涨跌幅后的日期列
|
||||
region_data['月份'] = region_data['original_date']
|
||||
|
||||
# 删除临时列
|
||||
region_data.drop(columns=['original_date'], inplace=True)
|
||||
|
||||
# 将区域数据添加到总的数据中
|
||||
all_region_data = pd.concat([all_region_data, region_data], ignore_index=True)
|
||||
|
||||
return all_region_data
|
||||
|
||||
|
||||
# 城市价个和区域价格比较
|
||||
def add_price_comparison_by_month(region_data):
|
||||
# 遍历区域数据,比较城市和区域的价格,并添加比值超过3倍的情况到区域数据中
|
||||
for index, row in region_data.iterrows():
|
||||
city = row['城市']
|
||||
region = row['区域']
|
||||
region_month = row['月份']
|
||||
region_price = row['估家计算均价']
|
||||
city_data = city_data_useinregion
|
||||
# 获取城市对应月份的价格
|
||||
city_price_row = city_data[(city_data['城市'] == city) & (city_data['月份'] == region_month)]
|
||||
if city_price_row.empty:
|
||||
continue
|
||||
city_price = city_price_row['估家城市均价'].iloc[0]
|
||||
|
||||
# 比较价格并计算比值
|
||||
price_ratio = max(city_price, region_price) / min(city_price, region_price)
|
||||
|
||||
# 如果比值超过3倍,将结果添加到区域数据中
|
||||
if price_ratio > 3:
|
||||
region_data.loc[index, '城市区域价格比值超3倍'] = price_ratio
|
||||
|
||||
return region_data
|
||||
|
||||
|
||||
def region_missing():
|
||||
df = region_increase(region_data, all_region_data)
|
||||
df['月份1'] = df['月份']
|
||||
# 将日期字符串转换为日期时间对象
|
||||
df["月份"] = pd.to_datetime(df["月份"], format="%Y%m")
|
||||
|
||||
# 获取当前时间
|
||||
current_time = pd.Timestamp.today().replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
|
||||
# 计算前一年到上月的时间范围
|
||||
one_year_ago = current_time - pd.DateOffset(years=1, days=current_time.day - 1)
|
||||
last_month = current_time - pd.DateOffset(months=1, days=current_time.day - 1)
|
||||
|
||||
# 创建一个包含所有可能月份的日期范围
|
||||
all_months = pd.date_range(start=one_year_ago, end=last_month, freq="MS")
|
||||
|
||||
# 获取每个城市-区域组合缺失的月份
|
||||
missing_months = []
|
||||
for (city, region), group in df.groupby(["城市", "区域"]):
|
||||
available_months = group["月份"]
|
||||
missing = all_months.difference(available_months)
|
||||
if len(missing) > 0:
|
||||
missing_str = ",".join([m.strftime("%Y%m") for m in missing])
|
||||
missing_months.append({"城市": city, "区域": region, "缺失月份": missing_str})
|
||||
|
||||
if missing_months:
|
||||
# 创建DataFrame来存储结果
|
||||
missing_df = pd.DataFrame(missing_months)
|
||||
path = pathAll+'/区域价格数据缺失'+year_month_day+'.xlsx'
|
||||
missing_df.to_excel(path, index=False)
|
||||
logging.info(f"区域价格缺失数据已写入Excel文件:{path}")
|
||||
cloudPath = '共享/统计/城市区域涨跌幅/' + '区域价格数据缺失' + year_month_day + '.xlsx'
|
||||
up_to_nextcloud(path, cloudPath)
|
||||
logging.info(f"区域价格缺失数据Excel文件已经上传到云盘")
|
||||
|
||||
# 如果区域在 missing_df 中,则将 df 中的该区域数据删除
|
||||
for city, region in zip(missing_df['城市'], missing_df['区域']):
|
||||
df = df[(df['城市'] != city) | (df['区域'] != region)]
|
||||
|
||||
df['月份'] = df['月份1']
|
||||
# 删除月份1列
|
||||
df.drop(columns=['月份1'], inplace=True)
|
||||
|
||||
# 添加价格比值超过3倍的列
|
||||
# print(city_data_useinregion)
|
||||
df = add_price_comparison_by_month(df)
|
||||
|
||||
output_path = pathAll+'/区域价格涨跌幅'+year_month_day+'.xlsx'
|
||||
df.to_excel(output_path, index=False)
|
||||
logging.info(f"区域涨跌幅数据已写入Excel文件:{output_path}")
|
||||
|
||||
cloudPath = '共享/统计/城市区域涨跌幅/' + '区域价格涨跌幅'+year_month_day+'.xlsx'
|
||||
up_to_nextcloud(output_path, cloudPath)
|
||||
logging.info(f"区域涨跌幅数据Excel文件已经上传到云盘")
|
||||
|
||||
|
||||
city_missing()
|
||||
region_missing()
|
||||
# """
|
||||
# 统计城市 区域 价格涨跌幅
|
||||
# """
|
||||
# import logging
|
||||
# import pandas as pd
|
||||
# from sqlalchemy import create_engine
|
||||
# from goojia_utils import dateutils
|
||||
# import datetime
|
||||
# import os
|
||||
# pathAll = "/home/airflow/data/cityPrice"
|
||||
# if not os.path.exists(pathAll):
|
||||
# os.makedirs(pathAll)
|
||||
# # 创建数据库引擎
|
||||
# engine235 = create_engine("postgresql+psycopg2://postgres:Goojia123456@192.168.0.235:5433/fangjia?client_encoding=utf8",
|
||||
# echo=False)
|
||||
# pd.set_option('mode.chained_assignment', None)
|
||||
# year_month_day = dateutils.format_date(datetime.datetime.now())
|
||||
# # 执行SQL查询获取城市价格数据
|
||||
# city_sql = "SELECT DISTINCT urbn_nm FROM price_stat.avm_city_price"
|
||||
# city_names_set = set(pd.read_sql(city_sql, engine235)['urbn_nm'])
|
||||
# city_names = list(city_names_set)
|
||||
# # 假设这是你的pgsql数据
|
||||
# all_city_data = pd.DataFrame()
|
||||
# # 执行SQL查询获取区域价格数据
|
||||
# region_sql = "SELECT DISTINCT urbn_nm,gxda_admndiv_nm FROM price_stat.avm_district_price"
|
||||
# region_data = pd.read_sql(region_sql, engine235)
|
||||
# city_data_useinregion = pd.DataFrame()
|
||||
# # 假设这是你的pgsql数据
|
||||
# all_region_data = pd.DataFrame()
|
||||
#
|
||||
# def up_to_nextcloud(aim_file_name,aim_file_path):
|
||||
# """
|
||||
# 上传统计信息至nextcloud
|
||||
# """
|
||||
# from nextcloud import NextCloud
|
||||
# NEXTCLOUD_URL = "https://nextcloud.fangjia.com"
|
||||
# NEXTCLOUD_USERNAME = "shenjian@fangjia.com"
|
||||
# NEXTCLOUD_PASSWORD = "goojia123456"
|
||||
# nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD, json_output=True)
|
||||
# nxc.upload_file(aim_file_name, aim_file_path)
|
||||
# def city_increase(city_names, all_city_data):
|
||||
# global city_data_useinregion # 声明为全局变量
|
||||
# # 遍历城市列表,对每个城市进行查询
|
||||
# for city in city_names:
|
||||
# query = f"SELECT urbn_nm as 城市, gujiabds_clc_yrmo_val as 月份, gujia_bds_clc_avprc as 估家计算均价, gujia_bds_urbn_avprc as 估家城市均价 FROM price_stat.avm_city_price WHERE urbn_nm = '{city}'"
|
||||
# city_data = pd.read_sql(query, engine235)
|
||||
#
|
||||
# if city_data.empty:
|
||||
# continue
|
||||
#
|
||||
# # 复制一份原始的日期列,用于计算涨跌幅时保持原始数据不变
|
||||
# city_data['original_date'] = city_data['月份']
|
||||
#
|
||||
# # 将日期列转换为日期类型
|
||||
# city_data['月份'] = pd.to_datetime(city_data['月份'], format='%Y%m')
|
||||
#
|
||||
# # 对价格数据按照月份排序
|
||||
# city_data.sort_values(by='月份', inplace=True)
|
||||
#
|
||||
# # 计算涨跌幅
|
||||
# city_data['涨跌幅'] = city_data['估家计算均价'].diff() / city_data['估家计算均价'].shift(1) * 100
|
||||
#
|
||||
# # 计算涨跌幅,并保留两位小数,展示为百分比形式
|
||||
# price_change = city_data['估家计算均价'].pct_change() * 100
|
||||
# city_data['涨跌幅'] = (price_change[price_change != 0].fillna(0).round(2).astype(str) + '%').replace('0.0%', '')
|
||||
#
|
||||
# # 将原始日期列替换为计算涨跌幅后的日期列
|
||||
# city_data['月份'] = city_data['original_date']
|
||||
#
|
||||
# # 删除临时列
|
||||
# city_data.drop(columns=['original_date'], inplace=True)
|
||||
#
|
||||
# # 将城市数据添加到总的数据中
|
||||
# all_city_data = pd.concat([all_city_data, city_data], ignore_index=True)
|
||||
# city_data_useinregion = all_city_data
|
||||
# return all_city_data
|
||||
#
|
||||
#
|
||||
# def city_missing():
|
||||
# df = city_increase(city_names, all_city_data)
|
||||
# df['月份1'] = df['月份']
|
||||
# # 将日期字符串转换为日期时间对象
|
||||
# df["月份"] = pd.to_datetime(df["月份"], format="%Y%m")
|
||||
#
|
||||
# # 获取当前时间
|
||||
# current_time = pd.Timestamp.today().replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
#
|
||||
# # 计算前一年到上月的时间范围
|
||||
# one_year_ago = current_time - pd.DateOffset(years=1, days=current_time.day - 1)
|
||||
# last_month = current_time - pd.DateOffset(months=1, days=current_time.day - 1)
|
||||
#
|
||||
# # 创建一个包含所有可能月份的日期范围
|
||||
# all_months = pd.date_range(start=one_year_ago, end=last_month, freq="MS")
|
||||
#
|
||||
# # 获取每个城市缺失的月份
|
||||
# missing_months = []
|
||||
# for city, group in df.groupby("城市"):
|
||||
# available_months = group["月份"]
|
||||
# missing = all_months.difference(available_months)
|
||||
# if len(missing) > 0:
|
||||
# missing_str = ",".join([m.strftime("%Y%m") for m in missing])
|
||||
# missing_months.append({"城市": city, "缺失月份": missing_str})
|
||||
# if len(missing_months) > 0:
|
||||
# # 创建DataFrame来存储结果
|
||||
# missing_df = pd.DataFrame(missing_months)
|
||||
# path = pathAll+'/城市价格数据缺失'+year_month_day+'.xlsx'
|
||||
# missing_df.to_excel(path, index=False)
|
||||
# logging.info(f"城市价格缺失数据已写入Excel文件:{path}")
|
||||
#
|
||||
# cloudPath = '共享/统计/城市区域涨跌幅/' + '城市价格数据缺失'+year_month_day+'.xlsx'
|
||||
# up_to_nextcloud(path, cloudPath)
|
||||
# logging.info(f"区域涨跌幅数据Excel文件已经上传到云盘")
|
||||
#
|
||||
# # print(missing_df)
|
||||
# df['月份'] = df['月份1']
|
||||
# # 删除月份1列
|
||||
# df.drop(columns=['月份1'], inplace=True)
|
||||
# output_path = pathAll+'/城市价格涨跌幅'+year_month_day+'.xlsx'
|
||||
# df.to_excel(output_path, index=False)
|
||||
# logging.info(f"城市涨跌幅数据已写入Excel文件:{output_path}")
|
||||
#
|
||||
# cloudPath = '共享/统计/城市区域涨跌幅/' + '城市价格涨跌幅'+year_month_day+'.xlsx'
|
||||
# up_to_nextcloud(output_path, cloudPath)
|
||||
# logging.info(f"区域涨跌幅数据Excel文件已经上传到云盘")
|
||||
#
|
||||
#
|
||||
# def region_increase(region_data, all_region_data):
|
||||
# # 遍历区域列表,对每个区域进行查询
|
||||
# for index, row in region_data.iterrows():
|
||||
# city = row['urbn_nm']
|
||||
# region = row['gxda_admndiv_nm']
|
||||
# query = f"SELECT urbn_nm as 城市, gxda_admndiv_nm as 区域, gujiabds_clc_yrmo_val as 月份, gujia_bds_clc_avprc as 估家计算均价, gujia_bds_urbn_avprc as 估家城市均价 FROM price_stat.avm_district_price WHERE urbn_nm = '{city}' AND gxda_admndiv_nm = '{region}'"
|
||||
# region_data = pd.read_sql(query, engine235)
|
||||
#
|
||||
# if region_data.empty:
|
||||
# continue
|
||||
#
|
||||
# # 复制一份原始的日期列,用于计算涨跌幅时保持原始数据不变
|
||||
# region_data['original_date'] = region_data['月份']
|
||||
#
|
||||
# # 将日期列转换为日期类型
|
||||
# region_data['月份'] = pd.to_datetime(region_data['月份'], format='%Y%m')
|
||||
#
|
||||
# # 对价格数据按照月份排序
|
||||
# region_data.sort_values(by='月份', inplace=True)
|
||||
#
|
||||
# # 计算涨跌幅
|
||||
# region_data['涨跌幅'] = region_data['估家计算均价'].diff() / region_data['估家计算均价'].shift(1) * 100
|
||||
#
|
||||
# # 计算涨跌幅,并保留两位小数,展示为百分比形式
|
||||
# price_change = region_data['估家计算均价'].pct_change() * 100
|
||||
# region_data['涨跌幅'] = (price_change[price_change != 0].fillna(0).round(2).astype(str) + '%').replace('0.0%',
|
||||
# '')
|
||||
#
|
||||
# # 将原始日期列替换为计算涨跌幅后的日期列
|
||||
# region_data['月份'] = region_data['original_date']
|
||||
#
|
||||
# # 删除临时列
|
||||
# region_data.drop(columns=['original_date'], inplace=True)
|
||||
#
|
||||
# # 将区域数据添加到总的数据中
|
||||
# all_region_data = pd.concat([all_region_data, region_data], ignore_index=True)
|
||||
#
|
||||
# return all_region_data
|
||||
#
|
||||
#
|
||||
# # 城市价个和区域价格比较
|
||||
# def add_price_comparison_by_month(region_data):
|
||||
# # 遍历区域数据,比较城市和区域的价格,并添加比值超过3倍的情况到区域数据中
|
||||
# for index, row in region_data.iterrows():
|
||||
# city = row['城市']
|
||||
# region = row['区域']
|
||||
# region_month = row['月份']
|
||||
# region_price = row['估家计算均价']
|
||||
# city_data = city_data_useinregion
|
||||
# # 获取城市对应月份的价格
|
||||
# city_price_row = city_data[(city_data['城市'] == city) & (city_data['月份'] == region_month)]
|
||||
# if city_price_row.empty:
|
||||
# continue
|
||||
# city_price = city_price_row['估家城市均价'].iloc[0]
|
||||
#
|
||||
# # 比较价格并计算比值
|
||||
# price_ratio = max(city_price, region_price) / min(city_price, region_price)
|
||||
#
|
||||
# # 如果比值超过3倍,将结果添加到区域数据中
|
||||
# if price_ratio > 3:
|
||||
# region_data.loc[index, '城市区域价格比值超3倍'] = price_ratio
|
||||
#
|
||||
# return region_data
|
||||
#
|
||||
#
|
||||
# def region_missing():
|
||||
# df = region_increase(region_data, all_region_data)
|
||||
# df['月份1'] = df['月份']
|
||||
# # 将日期字符串转换为日期时间对象
|
||||
# df["月份"] = pd.to_datetime(df["月份"], format="%Y%m")
|
||||
#
|
||||
# # 获取当前时间
|
||||
# current_time = pd.Timestamp.today().replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
#
|
||||
# # 计算前一年到上月的时间范围
|
||||
# one_year_ago = current_time - pd.DateOffset(years=1, days=current_time.day - 1)
|
||||
# last_month = current_time - pd.DateOffset(months=1, days=current_time.day - 1)
|
||||
#
|
||||
# # 创建一个包含所有可能月份的日期范围
|
||||
# all_months = pd.date_range(start=one_year_ago, end=last_month, freq="MS")
|
||||
#
|
||||
# # 获取每个城市-区域组合缺失的月份
|
||||
# missing_months = []
|
||||
# for (city, region), group in df.groupby(["城市", "区域"]):
|
||||
# available_months = group["月份"]
|
||||
# missing = all_months.difference(available_months)
|
||||
# if len(missing) > 0:
|
||||
# missing_str = ",".join([m.strftime("%Y%m") for m in missing])
|
||||
# missing_months.append({"城市": city, "区域": region, "缺失月份": missing_str})
|
||||
#
|
||||
# if missing_months:
|
||||
# # 创建DataFrame来存储结果
|
||||
# missing_df = pd.DataFrame(missing_months)
|
||||
# path = pathAll+'/区域价格数据缺失'+year_month_day+'.xlsx'
|
||||
# missing_df.to_excel(path, index=False)
|
||||
# logging.info(f"区域价格缺失数据已写入Excel文件:{path}")
|
||||
# cloudPath = '共享/统计/城市区域涨跌幅/' + '区域价格数据缺失' + year_month_day + '.xlsx'
|
||||
# up_to_nextcloud(path, cloudPath)
|
||||
# logging.info(f"区域价格缺失数据Excel文件已经上传到云盘")
|
||||
#
|
||||
# # 如果区域在 missing_df 中,则将 df 中的该区域数据删除
|
||||
# for city, region in zip(missing_df['城市'], missing_df['区域']):
|
||||
# df = df[(df['城市'] != city) | (df['区域'] != region)]
|
||||
#
|
||||
# df['月份'] = df['月份1']
|
||||
# # 删除月份1列
|
||||
# df.drop(columns=['月份1'], inplace=True)
|
||||
#
|
||||
# # 添加价格比值超过3倍的列
|
||||
# # print(city_data_useinregion)
|
||||
# df = add_price_comparison_by_month(df)
|
||||
#
|
||||
# output_path = pathAll+'/区域价格涨跌幅'+year_month_day+'.xlsx'
|
||||
# df.to_excel(output_path, index=False)
|
||||
# logging.info(f"区域涨跌幅数据已写入Excel文件:{output_path}")
|
||||
#
|
||||
# cloudPath = '共享/统计/城市区域涨跌幅/' + '区域价格涨跌幅'+year_month_day+'.xlsx'
|
||||
# up_to_nextcloud(output_path, cloudPath)
|
||||
# logging.info(f"区域涨跌幅数据Excel文件已经上传到云盘")
|
||||
#
|
||||
#
|
||||
# city_missing()
|
||||
# region_missing()
|
||||
|
||||
@@ -1,97 +0,0 @@
|
||||
import logging
|
||||
import pandas as pd
|
||||
from sqlalchemy import create_engine
|
||||
|
||||
# 创建数据库引擎
|
||||
engine235 = create_engine("postgresql+psycopg2://postgres:Goojia123456@192.168.0.235:5433/fangjia?client_encoding=utf8",
|
||||
echo=False)
|
||||
|
||||
# 执行SQL查询获取区域价格数据
|
||||
region_sql = "SELECT DISTINCT urbn_nm,gxda_admndiv_nm FROM price_stat.avm_district_price_copy1"
|
||||
region_data = pd.read_sql(region_sql, engine235)
|
||||
|
||||
# 假设这是你的pgsql数据
|
||||
all_region_data = pd.DataFrame()
|
||||
pd.set_option('mode.chained_assignment', None)
|
||||
|
||||
def region_increase(region_data, all_region_data):
|
||||
# 遍历区域列表,对每个区域进行查询
|
||||
for index, row in region_data.iterrows():
|
||||
city = row['urbn_nm']
|
||||
region = row['gxda_admndiv_nm']
|
||||
query = f"SELECT urbn_nm as 城市, gxda_admndiv_nm as 区域, gujiabds_clc_yrmo_val as 月份, gujia_bds_clc_avprc as 估家计算均价, gujia_bds_urbn_avprc as 估家城市均价 FROM price_stat.avm_district_price_copy1 WHERE urbn_nm = '{city}' AND gxda_admndiv_nm = '{region}'"
|
||||
region_data = pd.read_sql(query, engine235)
|
||||
|
||||
if region_data.empty:
|
||||
continue
|
||||
|
||||
# 复制一份原始的日期列,用于计算涨跌幅时保持原始数据不变
|
||||
region_data['original_date'] = region_data['月份']
|
||||
|
||||
# 将日期列转换为日期类型
|
||||
region_data['月份'] = pd.to_datetime(region_data['月份'], format='%Y%m')
|
||||
|
||||
# 对价格数据按照月份排序
|
||||
region_data.sort_values(by='月份', inplace=True)
|
||||
|
||||
# 计算涨跌幅
|
||||
region_data['涨跌幅'] = region_data['估家计算均价'].diff() / region_data['估家计算均价'].shift(1) * 100
|
||||
|
||||
# 计算涨跌幅,并保留两位小数,展示为百分比形式
|
||||
price_change = region_data['估家计算均价'].pct_change() * 100
|
||||
region_data['涨跌幅'] = (price_change[price_change != 0].fillna(0).round(2).astype(str) + '%').replace('0.0%', '')
|
||||
|
||||
# 将原始日期列替换为计算涨跌幅后的日期列
|
||||
region_data['月份'] = region_data['original_date']
|
||||
|
||||
# 删除临时列
|
||||
region_data.drop(columns=['original_date'], inplace=True)
|
||||
|
||||
# 将区域数据添加到总的数据中
|
||||
all_region_data = pd.concat([all_region_data, region_data], ignore_index=True)
|
||||
|
||||
return all_region_data
|
||||
|
||||
def region_missing():
|
||||
df = region_increase(region_data, all_region_data)
|
||||
df['月份1'] = df['月份']
|
||||
# 将日期字符串转换为日期时间对象
|
||||
df["月份"] = pd.to_datetime(df["月份"], format="%Y%m")
|
||||
|
||||
# 获取当前时间
|
||||
current_time = pd.Timestamp.today().replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
|
||||
# 计算前一年到上月的时间范围
|
||||
one_year_ago = current_time - pd.DateOffset(years=1, days=current_time.day - 1)
|
||||
last_month = current_time - pd.DateOffset(months=1, days=current_time.day - 1)
|
||||
|
||||
# 创建一个包含所有可能月份的日期范围
|
||||
all_months = pd.date_range(start=one_year_ago, end=last_month, freq="MS")
|
||||
|
||||
# 获取每个城市-区域组合缺失的月份
|
||||
missing_months = []
|
||||
for (city, region), group in df.groupby(["城市", "区域"]):
|
||||
available_months = group["月份"]
|
||||
missing = all_months.difference(available_months)
|
||||
if len(missing) > 0:
|
||||
missing_str = ",".join([m.strftime("%Y%m") for m in missing])
|
||||
missing_months.append({"城市": city, "区域": region, "缺失月份": missing_str})
|
||||
|
||||
if missing_months:
|
||||
# 创建DataFrame来存储结果
|
||||
missing_df = pd.DataFrame(missing_months)
|
||||
path = 'E:\\goojia\\file\\excel文件\\airflow\\missing_region.xlsx'
|
||||
missing_df.to_excel(path, index=False)
|
||||
|
||||
# 如果区域在 missing_df 中,则将 df 中的该区域数据删除
|
||||
for city, region in zip(missing_df['城市'], missing_df['区域']):
|
||||
df = df[(df['城市'] != city) | (df['区域'] != region)]
|
||||
|
||||
df['月份'] = df['月份1']
|
||||
# 删除月份1列
|
||||
df.drop(columns=['月份1'], inplace=True)
|
||||
output_path = 'E:\\goojia\\file\\excel文件\\airflow\\all_region_price_with_change.xlsx'
|
||||
df.to_excel(output_path, index=False)
|
||||
logging.info(f"区域涨跌幅数据已写入Excel文件:{output_path}")
|
||||
|
||||
region_missing()
|
||||
@@ -1,17 +1,17 @@
|
||||
from airflow.operators.bash import BashOperator
|
||||
import pendulum
|
||||
from airflow import DAG
|
||||
|
||||
with DAG(
|
||||
dag_id="increase_decrease_city_region_dag",
|
||||
# 每月的最后一天运行
|
||||
schedule="0 10 L * *",
|
||||
start_date=pendulum.datetime(2024, 5, 30, tz="Asia/Shanghai"),
|
||||
description="城市区域涨跌幅数据统计",
|
||||
catchup=True
|
||||
) as dag:
|
||||
t1 = BashOperator(
|
||||
task_id='increase_decrease_city_region',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/stat/cityPrice/increase_decrease_city_region.py',
|
||||
)
|
||||
t1
|
||||
# from airflow.operators.bash import BashOperator
|
||||
# import pendulum
|
||||
# from airflow import DAG
|
||||
#
|
||||
# with DAG(
|
||||
# dag_id="increase_decrease_city_region_dag",
|
||||
# # 每月的最后一天运行
|
||||
# schedule="0 10 L * *",
|
||||
# start_date=pendulum.datetime(2024, 5, 30, tz="Asia/Shanghai"),
|
||||
# description="城市区域涨跌幅数据统计",
|
||||
# catchup=True
|
||||
# ) as dag:
|
||||
# t1 = BashOperator(
|
||||
# task_id='increase_decrease_city_region',
|
||||
# bash_command='python /opt/airflow/dags/repo/dags/stat/cityPrice/increase_decrease_city_region.py',
|
||||
# )
|
||||
# t1
|
||||
|
||||
@@ -1,36 +0,0 @@
|
||||
from airflow import DAG
|
||||
from airflow.operators.python_operator import PythonOperator
|
||||
from datetime import datetime
|
||||
from goojia_utils import dingutils
|
||||
|
||||
# def send_dingding_alert(context):
|
||||
# print('监测到错误')
|
||||
# task_instance = context.get('task_instance')
|
||||
# url = task_instance.log_url.replace("http://localhost:8080", "https://airflow.fangjia.com")
|
||||
# message = f"【airflow】 任务{task_instance.task_id}错误,请点击{url}查看!"
|
||||
# print(message)
|
||||
# dingutils.send_text(message, "ae14f50045e9375c354940a2afb86f14e453bd1f08f78da0ba6e788d35bccec4")
|
||||
# print("success")
|
||||
#
|
||||
def fail_task():
|
||||
raise ValueError("This task is designed to fail")
|
||||
#
|
||||
# default_args = {
|
||||
# "owner": "airflow",
|
||||
# "retries": 0,
|
||||
# "on_failure_callback": send_dingding_alert, # 任务失败时触发钉钉告警
|
||||
# }
|
||||
|
||||
|
||||
dag = DAG(
|
||||
'test_failure_notification',
|
||||
#default_args=default_args,
|
||||
start_date=datetime(2025, 4, 22),
|
||||
schedule=None
|
||||
)
|
||||
|
||||
task = PythonOperator(
|
||||
task_id='failing_task',
|
||||
python_callable=fail_task,
|
||||
dag=dag
|
||||
)
|
||||
Reference in New Issue
Block a user