From 9467a62152ff5dc6e4615da1b41f86f9525863d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=96=87=E8=89=B3?= Date: Wed, 22 Jul 2026 11:41:49 +0800 Subject: [PATCH] =?UTF-8?q?=E7=89=88=E6=9C=AC=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/ai/dag_ai_stock.py | 46 +- dags/ai/stock/stock.py | 76 +-- dags/business/dag_comm_add.py | 118 ++--- dags/office/dag_office_rent.py | 2 +- dags/office/dag_office_sell.py | 4 +- .../increase_decrease_city_region.py | 498 +++++++++--------- dags/stat/cityPrice/test.py | 97 ---- .../stat/dag_increase_decrease_city_region.py | 34 +- dags/test/fail_test.py | 36 -- 9 files changed, 389 insertions(+), 522 deletions(-) delete mode 100644 dags/stat/cityPrice/test.py delete mode 100644 dags/test/fail_test.py diff --git a/dags/ai/dag_ai_stock.py b/dags/ai/dag_ai_stock.py index 3f5436b..ce0feea 100644 --- a/dags/ai/dag_ai_stock.py +++ b/dags/ai/dag_ai_stock.py @@ -1,23 +1,23 @@ -from airflow.operators.bash import BashOperator -import pendulum -from airflow import DAG - -with DAG( - dag_id="ai_stock_dag", - # 每天运行 - schedule="30 8 * * 1-5", - start_date=pendulum.datetime(2025, 11, 10, tz="Asia/Shanghai"), - catchup=True, - tags=["ai_stock"], -) as dag: - - t1 = BashOperator( - task_id='stock_strategy', - bash_command='python /opt/airflow/dags/repo/dags/ai/stock/stock_strategy.py', - ) - - t2 = BashOperator( - task_id='ai_stock', - bash_command='python /opt/airflow/dags/repo/dags/ai/stock/stock.py', - ) - t1 >> t2 \ No newline at end of file +# from airflow.operators.bash import BashOperator +# import pendulum +# from airflow import DAG +# +# with DAG( +# dag_id="ai_stock_dag", +# # 每天运行 +# schedule="30 8 * * 1-5", +# start_date=pendulum.datetime(2025, 11, 10, tz="Asia/Shanghai"), +# catchup=True, +# tags=["ai_stock"], +# ) as dag: +# +# t1 = BashOperator( +# task_id='stock_strategy', +# bash_command='python /opt/airflow/dags/repo/dags/ai/stock/stock_strategy.py', +# ) +# +# t2 = BashOperator( +# task_id='ai_stock', +# bash_command='python /opt/airflow/dags/repo/dags/ai/stock/stock.py', +# ) +# t1 >> t2 \ No newline at end of file diff --git a/dags/ai/stock/stock.py b/dags/ai/stock/stock.py index 7ea9bd6..a3c5c5a 100644 --- a/dags/ai/stock/stock.py +++ b/dags/ai/stock/stock.py @@ -1,38 +1,38 @@ -from goojia_utils import MysqlClient, ArkAIClient, dingutils -from goojia_utils import dateutils -from datetime import datetime -import requests - -db_util = MysqlClient( - host="192.168.0.93", - port=3306, - database="stock", - user="root", - password="Goojia123456" -) -date = int(dateutils.format_date(dateutils.add_months(datetime.today(), months=-1), fm='%Y%m%d')) -select_stocks = db_util.scan_by_sql_to_data_frame("select ts_code as 股票代码, stock_name as 股票名称,close as 股价, pe as PE,turnover_rate as 换手率 from mu_stocks_new where `trade_date`= (select max(`trade_date`) from mu_stocks_new)") -stat = db_util.scan_by_sql_to_data_frame(f"select ts_code as 股票代码, count(ts_code) as 命中次数 from mu_stocks_new where `trade_date` > {date} group by ts_code") -select_stocks = select_stocks.merge(stat, on='股票代码', how='left') -select_stocks['命中次数'] = select_stocks['命中次数'].fillna(0) -select_stocks = select_stocks.sort_values(by=['命中次数', 'PE'], ascending=[False, True]) -def df_to_nested_list(df): - """转换为嵌套的 Markdown 列表""" - lines = [] - i= 1 - for _, row in df.iterrows(): - lines.append(f"{i}. **{row['股票名称']}** ({row['股票代码']}) PE: {row['PE']} 股价: {row['股价']:.2f} 出现次数: {row['命中次数']} 换手率: {row['换手率']:.2f}") - i = i + 1 - return '\n'.join(lines) -data = { - "msgtype": "markdown", - "markdown": { - "title": dateutils.format_date(datetime.today(), fm='%Y%m%d') + "选股策略播报", - "text": df_to_nested_list(select_stocks) - } -} -access_token = "1e5f9aaeb90b091bc79652b52800cad2f1dee1ac0990149252511965f6d1c12a" -# 发送消息至群 -msg = requests.post(url='https://oapi.dingtalk.com/robot/send?access_token=' + access_token, json=data).json() -msg = requests.post(url='https://oapi.dingtalk.com/robot/send?access_token=45ac47a34dc5491535d77d643512244a85d91d7e74676e9c42446dc06357326e', json=data).json() -print(msg) \ No newline at end of file +# from goojia_utils import MysqlClient, ArkAIClient, dingutils +# from goojia_utils import dateutils +# from datetime import datetime +# import requests +# +# db_util = MysqlClient( +# host="192.168.0.93", +# port=3306, +# database="stock", +# user="root", +# password="Goojia123456" +# ) +# date = int(dateutils.format_date(dateutils.add_months(datetime.today(), months=-1), fm='%Y%m%d')) +# select_stocks = db_util.scan_by_sql_to_data_frame("select ts_code as 股票代码, stock_name as 股票名称,close as 股价, pe as PE,turnover_rate as 换手率 from mu_stocks_new where `trade_date`= (select max(`trade_date`) from mu_stocks_new)") +# stat = db_util.scan_by_sql_to_data_frame(f"select ts_code as 股票代码, count(ts_code) as 命中次数 from mu_stocks_new where `trade_date` > {date} group by ts_code") +# select_stocks = select_stocks.merge(stat, on='股票代码', how='left') +# select_stocks['命中次数'] = select_stocks['命中次数'].fillna(0) +# select_stocks = select_stocks.sort_values(by=['命中次数', 'PE'], ascending=[False, True]) +# def df_to_nested_list(df): +# """转换为嵌套的 Markdown 列表""" +# lines = [] +# i= 1 +# for _, row in df.iterrows(): +# lines.append(f"{i}. **{row['股票名称']}** ({row['股票代码']}) PE: {row['PE']} 股价: {row['股价']:.2f} 出现次数: {row['命中次数']} 换手率: {row['换手率']:.2f}") +# i = i + 1 +# return '\n'.join(lines) +# data = { +# "msgtype": "markdown", +# "markdown": { +# "title": dateutils.format_date(datetime.today(), fm='%Y%m%d') + "选股策略播报", +# "text": df_to_nested_list(select_stocks) +# } +# } +# access_token = "1e5f9aaeb90b091bc79652b52800cad2f1dee1ac0990149252511965f6d1c12a" +# # 发送消息至群 +# msg = requests.post(url='https://oapi.dingtalk.com/robot/send?access_token=' + access_token, json=data).json() +# msg = requests.post(url='https://oapi.dingtalk.com/robot/send?access_token=45ac47a34dc5491535d77d643512244a85d91d7e74676e9c42446dc06357326e', json=data).json() +# print(msg) \ No newline at end of file diff --git a/dags/business/dag_comm_add.py b/dags/business/dag_comm_add.py index 0813559..b071f5e 100644 --- a/dags/business/dag_comm_add.py +++ b/dags/business/dag_comm_add.py @@ -1,59 +1,59 @@ -from airflow.providers.docker.operators.docker import DockerOperator -from airflow.providers.cncf.kubernetes.operators.kubernetes_pod import KubernetesPodOperator -import pendulum -from airflow import DAG -from docker.types import Mount -from kubernetes.client import models as k8s - -with DAG( - dag_id="comm_add_dag", - # 每月20日1点整分运行 - schedule="0 1 20 * *", - start_date=pendulum.datetime(2023, 5, 8, tz="Asia/Shanghai"), - catchup=True, - description="小区自动化入库" -) as dag: - # k = KubernetesPodOperator( - # name="comm-add", - # image="192.168.0.19:9012/comm-add:1.0.1", - # cmds=["bash", "-cx"], - # arguments=["echo", "10"], - # labels={"foo": "bar"}, - # task_id="comm_add", - # do_xcom_push=True, - # ) - # k.dry_run() - - production_task = KubernetesPodOperator(namespace='airflow-plus', - # image="my-production-job:release-1.0.1", <-- old release - image="192.168.0.19:9012/comm-add:1.0.1", - image_pull_policy="Always", - image_pull_secrets=[k8s.V1LocalObjectReference("nexus-registry-secret")], - cmds=["python", "main.py"], - arguments=["--city", '{{dag_run.conf["city"] if "city" in dag_run.conf else "all"}}'], - name="comm_add", - task_id="comm_add", - get_logs=True - ) - - production_task - # t1 = DockerOperator( - # # 刚才定义的conn id - # docker_conn_id="docker_repo", - # # WebServer中点击Admin -> Providers -> Docker对应的Version,如果是auto,可能会报错 - # api_version="2.2.0", - # auto_remove=True, - # docker_url='unix://var/run/docker.sock', - # network_mode='bridge', - # image='192.168.0.19:9012/comm-add:1.0.1', - # # 临沂为我们的传参,python代码中, 镜像中部分代码 - # # @ click.command() - # # @ click.option('--city', default="all", help='要计算的城市') - # #command='python main.py --city {{dag_run.conf["city"] if "city" in dag_run.conf}} else all', - # command='echo "this is a test message shown from within the container', - # mounts=[ - # Mount(source="/var/run/docker.sock", target="/var/run/docker.sock", type="bind") - # ], - # task_id='comm_add' - # ) - # t1 \ No newline at end of file +# from airflow.providers.docker.operators.docker import DockerOperator +# from airflow.providers.cncf.kubernetes.operators.kubernetes_pod import KubernetesPodOperator +# import pendulum +# from airflow import DAG +# from docker.types import Mount +# from kubernetes.client import models as k8s +# +# with DAG( +# dag_id="comm_add_dag", +# # 每月20日1点整分运行 +# schedule="0 1 20 * *", +# start_date=pendulum.datetime(2023, 5, 8, tz="Asia/Shanghai"), +# catchup=True, +# description="小区自动化入库" +# ) as dag: +# # k = KubernetesPodOperator( +# # name="comm-add", +# # image="192.168.0.19:9012/comm-add:1.0.1", +# # cmds=["bash", "-cx"], +# # arguments=["echo", "10"], +# # labels={"foo": "bar"}, +# # task_id="comm_add", +# # do_xcom_push=True, +# # ) +# # k.dry_run() +# +# production_task = KubernetesPodOperator(namespace='airflow-plus', +# # image="my-production-job:release-1.0.1", <-- old release +# image="192.168.0.19:9012/comm-add:1.0.1", +# image_pull_policy="Always", +# image_pull_secrets=[k8s.V1LocalObjectReference("nexus-registry-secret")], +# cmds=["python", "main.py"], +# arguments=["--city", '{{dag_run.conf["city"] if "city" in dag_run.conf else "all"}}'], +# name="comm_add", +# task_id="comm_add", +# get_logs=True +# ) +# +# production_task +# # t1 = DockerOperator( +# # # 刚才定义的conn id +# # docker_conn_id="docker_repo", +# # # WebServer中点击Admin -> Providers -> Docker对应的Version,如果是auto,可能会报错 +# # api_version="2.2.0", +# # auto_remove=True, +# # docker_url='unix://var/run/docker.sock', +# # network_mode='bridge', +# # image='192.168.0.19:9012/comm-add:1.0.1', +# # # 临沂为我们的传参,python代码中, 镜像中部分代码 +# # # @ click.command() +# # # @ click.option('--city', default="all", help='要计算的城市') +# # #command='python main.py --city {{dag_run.conf["city"] if "city" in dag_run.conf}} else all', +# # command='echo "this is a test message shown from within the container', +# # mounts=[ +# # Mount(source="/var/run/docker.sock", target="/var/run/docker.sock", type="bind") +# # ], +# # task_id='comm_add' +# # ) +# # t1 \ No newline at end of file diff --git a/dags/office/dag_office_rent.py b/dags/office/dag_office_rent.py index b202cb5..a8f935c 100644 --- a/dags/office/dag_office_rent.py +++ b/dags/office/dag_office_rent.py @@ -3,7 +3,7 @@ """ from airflow.operators.bash import BashOperator from airflow.operators.python import PythonOperator, BranchPythonOperator -from office.rent.base_price import calc +from rent.base_price import calc import pendulum from airflow import DAG diff --git a/dags/office/dag_office_sell.py b/dags/office/dag_office_sell.py index a2de014..3f5f807 100644 --- a/dags/office/dag_office_sell.py +++ b/dags/office/dag_office_sell.py @@ -1,8 +1,8 @@ from airflow.operators.bash import BashOperator from airflow.operators.python import PythonOperator, BranchPythonOperator from airflow.operators.empty import EmptyOperator -from office.sell.base_price import calc -from office.sell.fill_base_price import fill_base_price +from sell.base_price import calc +from sell.fill_base_price import fill_base_price import pendulum from airflow import DAG diff --git a/dags/stat/cityPrice/increase_decrease_city_region.py b/dags/stat/cityPrice/increase_decrease_city_region.py index 36036c3..c61501a 100644 --- a/dags/stat/cityPrice/increase_decrease_city_region.py +++ b/dags/stat/cityPrice/increase_decrease_city_region.py @@ -1,249 +1,249 @@ -""" -统计城市 区域 价格涨跌幅 -""" -import logging -import pandas as pd -from sqlalchemy import create_engine -from goojia_utils import dateutils -import datetime -import os -pathAll = "/home/airflow/data/cityPrice" -if not os.path.exists(pathAll): - os.makedirs(pathAll) -# 创建数据库引擎 -engine235 = create_engine("postgresql+psycopg2://postgres:Goojia123456@192.168.0.235:5433/fangjia?client_encoding=utf8", - echo=False) -pd.set_option('mode.chained_assignment', None) -year_month_day = dateutils.format_date(datetime.datetime.now()) -# 执行SQL查询获取城市价格数据 -city_sql = "SELECT DISTINCT urbn_nm FROM price_stat.avm_city_price" -city_names_set = set(pd.read_sql(city_sql, engine235)['urbn_nm']) -city_names = list(city_names_set) -# 假设这是你的pgsql数据 -all_city_data = pd.DataFrame() -# 执行SQL查询获取区域价格数据 -region_sql = "SELECT DISTINCT urbn_nm,gxda_admndiv_nm FROM price_stat.avm_district_price" -region_data = pd.read_sql(region_sql, engine235) -city_data_useinregion = pd.DataFrame() -# 假设这是你的pgsql数据 -all_region_data = pd.DataFrame() - -def up_to_nextcloud(aim_file_name,aim_file_path): - """ - 上传统计信息至nextcloud - """ - from nextcloud import NextCloud - NEXTCLOUD_URL = "https://nextcloud.fangjia.com" - NEXTCLOUD_USERNAME = "shenjian@fangjia.com" - NEXTCLOUD_PASSWORD = "goojia123456" - nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD, json_output=True) - nxc.upload_file(aim_file_name, aim_file_path) -def city_increase(city_names, all_city_data): - global city_data_useinregion # 声明为全局变量 - # 遍历城市列表,对每个城市进行查询 - for city in city_names: - query = f"SELECT urbn_nm as 城市, gujiabds_clc_yrmo_val as 月份, gujia_bds_clc_avprc as 估家计算均价, gujia_bds_urbn_avprc as 估家城市均价 FROM price_stat.avm_city_price WHERE urbn_nm = '{city}'" - city_data = pd.read_sql(query, engine235) - - if city_data.empty: - continue - - # 复制一份原始的日期列,用于计算涨跌幅时保持原始数据不变 - city_data['original_date'] = city_data['月份'] - - # 将日期列转换为日期类型 - city_data['月份'] = pd.to_datetime(city_data['月份'], format='%Y%m') - - # 对价格数据按照月份排序 - city_data.sort_values(by='月份', inplace=True) - - # 计算涨跌幅 - city_data['涨跌幅'] = city_data['估家计算均价'].diff() / city_data['估家计算均价'].shift(1) * 100 - - # 计算涨跌幅,并保留两位小数,展示为百分比形式 - price_change = city_data['估家计算均价'].pct_change() * 100 - city_data['涨跌幅'] = (price_change[price_change != 0].fillna(0).round(2).astype(str) + '%').replace('0.0%', '') - - # 将原始日期列替换为计算涨跌幅后的日期列 - city_data['月份'] = city_data['original_date'] - - # 删除临时列 - city_data.drop(columns=['original_date'], inplace=True) - - # 将城市数据添加到总的数据中 - all_city_data = pd.concat([all_city_data, city_data], ignore_index=True) - city_data_useinregion = all_city_data - return all_city_data - - -def city_missing(): - df = city_increase(city_names, all_city_data) - df['月份1'] = df['月份'] - # 将日期字符串转换为日期时间对象 - df["月份"] = pd.to_datetime(df["月份"], format="%Y%m") - - # 获取当前时间 - current_time = pd.Timestamp.today().replace(hour=0, minute=0, second=0, microsecond=0) - - # 计算前一年到上月的时间范围 - one_year_ago = current_time - pd.DateOffset(years=1, days=current_time.day - 1) - last_month = current_time - pd.DateOffset(months=1, days=current_time.day - 1) - - # 创建一个包含所有可能月份的日期范围 - all_months = pd.date_range(start=one_year_ago, end=last_month, freq="MS") - - # 获取每个城市缺失的月份 - missing_months = [] - for city, group in df.groupby("城市"): - available_months = group["月份"] - missing = all_months.difference(available_months) - if len(missing) > 0: - missing_str = ",".join([m.strftime("%Y%m") for m in missing]) - missing_months.append({"城市": city, "缺失月份": missing_str}) - if len(missing_months) > 0: - # 创建DataFrame来存储结果 - missing_df = pd.DataFrame(missing_months) - path = pathAll+'/城市价格数据缺失'+year_month_day+'.xlsx' - missing_df.to_excel(path, index=False) - logging.info(f"城市价格缺失数据已写入Excel文件:{path}") - - cloudPath = '共享/统计/城市区域涨跌幅/' + '城市价格数据缺失'+year_month_day+'.xlsx' - up_to_nextcloud(path, cloudPath) - logging.info(f"区域涨跌幅数据Excel文件已经上传到云盘") - - # print(missing_df) - df['月份'] = df['月份1'] - # 删除月份1列 - df.drop(columns=['月份1'], inplace=True) - output_path = pathAll+'/城市价格涨跌幅'+year_month_day+'.xlsx' - df.to_excel(output_path, index=False) - logging.info(f"城市涨跌幅数据已写入Excel文件:{output_path}") - - cloudPath = '共享/统计/城市区域涨跌幅/' + '城市价格涨跌幅'+year_month_day+'.xlsx' - up_to_nextcloud(output_path, cloudPath) - logging.info(f"区域涨跌幅数据Excel文件已经上传到云盘") - - -def region_increase(region_data, all_region_data): - # 遍历区域列表,对每个区域进行查询 - for index, row in region_data.iterrows(): - city = row['urbn_nm'] - region = row['gxda_admndiv_nm'] - query = f"SELECT urbn_nm as 城市, gxda_admndiv_nm as 区域, gujiabds_clc_yrmo_val as 月份, gujia_bds_clc_avprc as 估家计算均价, gujia_bds_urbn_avprc as 估家城市均价 FROM price_stat.avm_district_price WHERE urbn_nm = '{city}' AND gxda_admndiv_nm = '{region}'" - region_data = pd.read_sql(query, engine235) - - if region_data.empty: - continue - - # 复制一份原始的日期列,用于计算涨跌幅时保持原始数据不变 - region_data['original_date'] = region_data['月份'] - - # 将日期列转换为日期类型 - region_data['月份'] = pd.to_datetime(region_data['月份'], format='%Y%m') - - # 对价格数据按照月份排序 - region_data.sort_values(by='月份', inplace=True) - - # 计算涨跌幅 - region_data['涨跌幅'] = region_data['估家计算均价'].diff() / region_data['估家计算均价'].shift(1) * 100 - - # 计算涨跌幅,并保留两位小数,展示为百分比形式 - price_change = region_data['估家计算均价'].pct_change() * 100 - region_data['涨跌幅'] = (price_change[price_change != 0].fillna(0).round(2).astype(str) + '%').replace('0.0%', - '') - - # 将原始日期列替换为计算涨跌幅后的日期列 - region_data['月份'] = region_data['original_date'] - - # 删除临时列 - region_data.drop(columns=['original_date'], inplace=True) - - # 将区域数据添加到总的数据中 - all_region_data = pd.concat([all_region_data, region_data], ignore_index=True) - - return all_region_data - - -# 城市价个和区域价格比较 -def add_price_comparison_by_month(region_data): - # 遍历区域数据,比较城市和区域的价格,并添加比值超过3倍的情况到区域数据中 - for index, row in region_data.iterrows(): - city = row['城市'] - region = row['区域'] - region_month = row['月份'] - region_price = row['估家计算均价'] - city_data = city_data_useinregion - # 获取城市对应月份的价格 - city_price_row = city_data[(city_data['城市'] == city) & (city_data['月份'] == region_month)] - if city_price_row.empty: - continue - city_price = city_price_row['估家城市均价'].iloc[0] - - # 比较价格并计算比值 - price_ratio = max(city_price, region_price) / min(city_price, region_price) - - # 如果比值超过3倍,将结果添加到区域数据中 - if price_ratio > 3: - region_data.loc[index, '城市区域价格比值超3倍'] = price_ratio - - return region_data - - -def region_missing(): - df = region_increase(region_data, all_region_data) - df['月份1'] = df['月份'] - # 将日期字符串转换为日期时间对象 - df["月份"] = pd.to_datetime(df["月份"], format="%Y%m") - - # 获取当前时间 - current_time = pd.Timestamp.today().replace(hour=0, minute=0, second=0, microsecond=0) - - # 计算前一年到上月的时间范围 - one_year_ago = current_time - pd.DateOffset(years=1, days=current_time.day - 1) - last_month = current_time - pd.DateOffset(months=1, days=current_time.day - 1) - - # 创建一个包含所有可能月份的日期范围 - all_months = pd.date_range(start=one_year_ago, end=last_month, freq="MS") - - # 获取每个城市-区域组合缺失的月份 - missing_months = [] - for (city, region), group in df.groupby(["城市", "区域"]): - available_months = group["月份"] - missing = all_months.difference(available_months) - if len(missing) > 0: - missing_str = ",".join([m.strftime("%Y%m") for m in missing]) - missing_months.append({"城市": city, "区域": region, "缺失月份": missing_str}) - - if missing_months: - # 创建DataFrame来存储结果 - missing_df = pd.DataFrame(missing_months) - path = pathAll+'/区域价格数据缺失'+year_month_day+'.xlsx' - missing_df.to_excel(path, index=False) - logging.info(f"区域价格缺失数据已写入Excel文件:{path}") - cloudPath = '共享/统计/城市区域涨跌幅/' + '区域价格数据缺失' + year_month_day + '.xlsx' - up_to_nextcloud(path, cloudPath) - logging.info(f"区域价格缺失数据Excel文件已经上传到云盘") - - # 如果区域在 missing_df 中,则将 df 中的该区域数据删除 - for city, region in zip(missing_df['城市'], missing_df['区域']): - df = df[(df['城市'] != city) | (df['区域'] != region)] - - df['月份'] = df['月份1'] - # 删除月份1列 - df.drop(columns=['月份1'], inplace=True) - - # 添加价格比值超过3倍的列 - # print(city_data_useinregion) - df = add_price_comparison_by_month(df) - - output_path = pathAll+'/区域价格涨跌幅'+year_month_day+'.xlsx' - df.to_excel(output_path, index=False) - logging.info(f"区域涨跌幅数据已写入Excel文件:{output_path}") - - cloudPath = '共享/统计/城市区域涨跌幅/' + '区域价格涨跌幅'+year_month_day+'.xlsx' - up_to_nextcloud(output_path, cloudPath) - logging.info(f"区域涨跌幅数据Excel文件已经上传到云盘") - - -city_missing() -region_missing() +# """ +# 统计城市 区域 价格涨跌幅 +# """ +# import logging +# import pandas as pd +# from sqlalchemy import create_engine +# from goojia_utils import dateutils +# import datetime +# import os +# pathAll = "/home/airflow/data/cityPrice" +# if not os.path.exists(pathAll): +# os.makedirs(pathAll) +# # 创建数据库引擎 +# engine235 = create_engine("postgresql+psycopg2://postgres:Goojia123456@192.168.0.235:5433/fangjia?client_encoding=utf8", +# echo=False) +# pd.set_option('mode.chained_assignment', None) +# year_month_day = dateutils.format_date(datetime.datetime.now()) +# # 执行SQL查询获取城市价格数据 +# city_sql = "SELECT DISTINCT urbn_nm FROM price_stat.avm_city_price" +# city_names_set = set(pd.read_sql(city_sql, engine235)['urbn_nm']) +# city_names = list(city_names_set) +# # 假设这是你的pgsql数据 +# all_city_data = pd.DataFrame() +# # 执行SQL查询获取区域价格数据 +# region_sql = "SELECT DISTINCT urbn_nm,gxda_admndiv_nm FROM price_stat.avm_district_price" +# region_data = pd.read_sql(region_sql, engine235) +# city_data_useinregion = pd.DataFrame() +# # 假设这是你的pgsql数据 +# all_region_data = pd.DataFrame() +# +# def up_to_nextcloud(aim_file_name,aim_file_path): +# """ +# 上传统计信息至nextcloud +# """ +# from nextcloud import NextCloud +# NEXTCLOUD_URL = "https://nextcloud.fangjia.com" +# NEXTCLOUD_USERNAME = "shenjian@fangjia.com" +# NEXTCLOUD_PASSWORD = "goojia123456" +# nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD, json_output=True) +# nxc.upload_file(aim_file_name, aim_file_path) +# def city_increase(city_names, all_city_data): +# global city_data_useinregion # 声明为全局变量 +# # 遍历城市列表,对每个城市进行查询 +# for city in city_names: +# query = f"SELECT urbn_nm as 城市, gujiabds_clc_yrmo_val as 月份, gujia_bds_clc_avprc as 估家计算均价, gujia_bds_urbn_avprc as 估家城市均价 FROM price_stat.avm_city_price WHERE urbn_nm = '{city}'" +# city_data = pd.read_sql(query, engine235) +# +# if city_data.empty: +# continue +# +# # 复制一份原始的日期列,用于计算涨跌幅时保持原始数据不变 +# city_data['original_date'] = city_data['月份'] +# +# # 将日期列转换为日期类型 +# city_data['月份'] = pd.to_datetime(city_data['月份'], format='%Y%m') +# +# # 对价格数据按照月份排序 +# city_data.sort_values(by='月份', inplace=True) +# +# # 计算涨跌幅 +# city_data['涨跌幅'] = city_data['估家计算均价'].diff() / city_data['估家计算均价'].shift(1) * 100 +# +# # 计算涨跌幅,并保留两位小数,展示为百分比形式 +# price_change = city_data['估家计算均价'].pct_change() * 100 +# city_data['涨跌幅'] = (price_change[price_change != 0].fillna(0).round(2).astype(str) + '%').replace('0.0%', '') +# +# # 将原始日期列替换为计算涨跌幅后的日期列 +# city_data['月份'] = city_data['original_date'] +# +# # 删除临时列 +# city_data.drop(columns=['original_date'], inplace=True) +# +# # 将城市数据添加到总的数据中 +# all_city_data = pd.concat([all_city_data, city_data], ignore_index=True) +# city_data_useinregion = all_city_data +# return all_city_data +# +# +# def city_missing(): +# df = city_increase(city_names, all_city_data) +# df['月份1'] = df['月份'] +# # 将日期字符串转换为日期时间对象 +# df["月份"] = pd.to_datetime(df["月份"], format="%Y%m") +# +# # 获取当前时间 +# current_time = pd.Timestamp.today().replace(hour=0, minute=0, second=0, microsecond=0) +# +# # 计算前一年到上月的时间范围 +# one_year_ago = current_time - pd.DateOffset(years=1, days=current_time.day - 1) +# last_month = current_time - pd.DateOffset(months=1, days=current_time.day - 1) +# +# # 创建一个包含所有可能月份的日期范围 +# all_months = pd.date_range(start=one_year_ago, end=last_month, freq="MS") +# +# # 获取每个城市缺失的月份 +# missing_months = [] +# for city, group in df.groupby("城市"): +# available_months = group["月份"] +# missing = all_months.difference(available_months) +# if len(missing) > 0: +# missing_str = ",".join([m.strftime("%Y%m") for m in missing]) +# missing_months.append({"城市": city, "缺失月份": missing_str}) +# if len(missing_months) > 0: +# # 创建DataFrame来存储结果 +# missing_df = pd.DataFrame(missing_months) +# path = pathAll+'/城市价格数据缺失'+year_month_day+'.xlsx' +# missing_df.to_excel(path, index=False) +# logging.info(f"城市价格缺失数据已写入Excel文件:{path}") +# +# cloudPath = '共享/统计/城市区域涨跌幅/' + '城市价格数据缺失'+year_month_day+'.xlsx' +# up_to_nextcloud(path, cloudPath) +# logging.info(f"区域涨跌幅数据Excel文件已经上传到云盘") +# +# # print(missing_df) +# df['月份'] = df['月份1'] +# # 删除月份1列 +# df.drop(columns=['月份1'], inplace=True) +# output_path = pathAll+'/城市价格涨跌幅'+year_month_day+'.xlsx' +# df.to_excel(output_path, index=False) +# logging.info(f"城市涨跌幅数据已写入Excel文件:{output_path}") +# +# cloudPath = '共享/统计/城市区域涨跌幅/' + '城市价格涨跌幅'+year_month_day+'.xlsx' +# up_to_nextcloud(output_path, cloudPath) +# logging.info(f"区域涨跌幅数据Excel文件已经上传到云盘") +# +# +# def region_increase(region_data, all_region_data): +# # 遍历区域列表,对每个区域进行查询 +# for index, row in region_data.iterrows(): +# city = row['urbn_nm'] +# region = row['gxda_admndiv_nm'] +# query = f"SELECT urbn_nm as 城市, gxda_admndiv_nm as 区域, gujiabds_clc_yrmo_val as 月份, gujia_bds_clc_avprc as 估家计算均价, gujia_bds_urbn_avprc as 估家城市均价 FROM price_stat.avm_district_price WHERE urbn_nm = '{city}' AND gxda_admndiv_nm = '{region}'" +# region_data = pd.read_sql(query, engine235) +# +# if region_data.empty: +# continue +# +# # 复制一份原始的日期列,用于计算涨跌幅时保持原始数据不变 +# region_data['original_date'] = region_data['月份'] +# +# # 将日期列转换为日期类型 +# region_data['月份'] = pd.to_datetime(region_data['月份'], format='%Y%m') +# +# # 对价格数据按照月份排序 +# region_data.sort_values(by='月份', inplace=True) +# +# # 计算涨跌幅 +# region_data['涨跌幅'] = region_data['估家计算均价'].diff() / region_data['估家计算均价'].shift(1) * 100 +# +# # 计算涨跌幅,并保留两位小数,展示为百分比形式 +# price_change = region_data['估家计算均价'].pct_change() * 100 +# region_data['涨跌幅'] = (price_change[price_change != 0].fillna(0).round(2).astype(str) + '%').replace('0.0%', +# '') +# +# # 将原始日期列替换为计算涨跌幅后的日期列 +# region_data['月份'] = region_data['original_date'] +# +# # 删除临时列 +# region_data.drop(columns=['original_date'], inplace=True) +# +# # 将区域数据添加到总的数据中 +# all_region_data = pd.concat([all_region_data, region_data], ignore_index=True) +# +# return all_region_data +# +# +# # 城市价个和区域价格比较 +# def add_price_comparison_by_month(region_data): +# # 遍历区域数据,比较城市和区域的价格,并添加比值超过3倍的情况到区域数据中 +# for index, row in region_data.iterrows(): +# city = row['城市'] +# region = row['区域'] +# region_month = row['月份'] +# region_price = row['估家计算均价'] +# city_data = city_data_useinregion +# # 获取城市对应月份的价格 +# city_price_row = city_data[(city_data['城市'] == city) & (city_data['月份'] == region_month)] +# if city_price_row.empty: +# continue +# city_price = city_price_row['估家城市均价'].iloc[0] +# +# # 比较价格并计算比值 +# price_ratio = max(city_price, region_price) / min(city_price, region_price) +# +# # 如果比值超过3倍,将结果添加到区域数据中 +# if price_ratio > 3: +# region_data.loc[index, '城市区域价格比值超3倍'] = price_ratio +# +# return region_data +# +# +# def region_missing(): +# df = region_increase(region_data, all_region_data) +# df['月份1'] = df['月份'] +# # 将日期字符串转换为日期时间对象 +# df["月份"] = pd.to_datetime(df["月份"], format="%Y%m") +# +# # 获取当前时间 +# current_time = pd.Timestamp.today().replace(hour=0, minute=0, second=0, microsecond=0) +# +# # 计算前一年到上月的时间范围 +# one_year_ago = current_time - pd.DateOffset(years=1, days=current_time.day - 1) +# last_month = current_time - pd.DateOffset(months=1, days=current_time.day - 1) +# +# # 创建一个包含所有可能月份的日期范围 +# all_months = pd.date_range(start=one_year_ago, end=last_month, freq="MS") +# +# # 获取每个城市-区域组合缺失的月份 +# missing_months = [] +# for (city, region), group in df.groupby(["城市", "区域"]): +# available_months = group["月份"] +# missing = all_months.difference(available_months) +# if len(missing) > 0: +# missing_str = ",".join([m.strftime("%Y%m") for m in missing]) +# missing_months.append({"城市": city, "区域": region, "缺失月份": missing_str}) +# +# if missing_months: +# # 创建DataFrame来存储结果 +# missing_df = pd.DataFrame(missing_months) +# path = pathAll+'/区域价格数据缺失'+year_month_day+'.xlsx' +# missing_df.to_excel(path, index=False) +# logging.info(f"区域价格缺失数据已写入Excel文件:{path}") +# cloudPath = '共享/统计/城市区域涨跌幅/' + '区域价格数据缺失' + year_month_day + '.xlsx' +# up_to_nextcloud(path, cloudPath) +# logging.info(f"区域价格缺失数据Excel文件已经上传到云盘") +# +# # 如果区域在 missing_df 中,则将 df 中的该区域数据删除 +# for city, region in zip(missing_df['城市'], missing_df['区域']): +# df = df[(df['城市'] != city) | (df['区域'] != region)] +# +# df['月份'] = df['月份1'] +# # 删除月份1列 +# df.drop(columns=['月份1'], inplace=True) +# +# # 添加价格比值超过3倍的列 +# # print(city_data_useinregion) +# df = add_price_comparison_by_month(df) +# +# output_path = pathAll+'/区域价格涨跌幅'+year_month_day+'.xlsx' +# df.to_excel(output_path, index=False) +# logging.info(f"区域涨跌幅数据已写入Excel文件:{output_path}") +# +# cloudPath = '共享/统计/城市区域涨跌幅/' + '区域价格涨跌幅'+year_month_day+'.xlsx' +# up_to_nextcloud(output_path, cloudPath) +# logging.info(f"区域涨跌幅数据Excel文件已经上传到云盘") +# +# +# city_missing() +# region_missing() diff --git a/dags/stat/cityPrice/test.py b/dags/stat/cityPrice/test.py deleted file mode 100644 index 9b25db3..0000000 --- a/dags/stat/cityPrice/test.py +++ /dev/null @@ -1,97 +0,0 @@ -import logging -import pandas as pd -from sqlalchemy import create_engine - -# 创建数据库引擎 -engine235 = create_engine("postgresql+psycopg2://postgres:Goojia123456@192.168.0.235:5433/fangjia?client_encoding=utf8", - echo=False) - -# 执行SQL查询获取区域价格数据 -region_sql = "SELECT DISTINCT urbn_nm,gxda_admndiv_nm FROM price_stat.avm_district_price_copy1" -region_data = pd.read_sql(region_sql, engine235) - -# 假设这是你的pgsql数据 -all_region_data = pd.DataFrame() -pd.set_option('mode.chained_assignment', None) - -def region_increase(region_data, all_region_data): - # 遍历区域列表,对每个区域进行查询 - for index, row in region_data.iterrows(): - city = row['urbn_nm'] - region = row['gxda_admndiv_nm'] - query = f"SELECT urbn_nm as 城市, gxda_admndiv_nm as 区域, gujiabds_clc_yrmo_val as 月份, gujia_bds_clc_avprc as 估家计算均价, gujia_bds_urbn_avprc as 估家城市均价 FROM price_stat.avm_district_price_copy1 WHERE urbn_nm = '{city}' AND gxda_admndiv_nm = '{region}'" - region_data = pd.read_sql(query, engine235) - - if region_data.empty: - continue - - # 复制一份原始的日期列,用于计算涨跌幅时保持原始数据不变 - region_data['original_date'] = region_data['月份'] - - # 将日期列转换为日期类型 - region_data['月份'] = pd.to_datetime(region_data['月份'], format='%Y%m') - - # 对价格数据按照月份排序 - region_data.sort_values(by='月份', inplace=True) - - # 计算涨跌幅 - region_data['涨跌幅'] = region_data['估家计算均价'].diff() / region_data['估家计算均价'].shift(1) * 100 - - # 计算涨跌幅,并保留两位小数,展示为百分比形式 - price_change = region_data['估家计算均价'].pct_change() * 100 - region_data['涨跌幅'] = (price_change[price_change != 0].fillna(0).round(2).astype(str) + '%').replace('0.0%', '') - - # 将原始日期列替换为计算涨跌幅后的日期列 - region_data['月份'] = region_data['original_date'] - - # 删除临时列 - region_data.drop(columns=['original_date'], inplace=True) - - # 将区域数据添加到总的数据中 - all_region_data = pd.concat([all_region_data, region_data], ignore_index=True) - - return all_region_data - -def region_missing(): - df = region_increase(region_data, all_region_data) - df['月份1'] = df['月份'] - # 将日期字符串转换为日期时间对象 - df["月份"] = pd.to_datetime(df["月份"], format="%Y%m") - - # 获取当前时间 - current_time = pd.Timestamp.today().replace(hour=0, minute=0, second=0, microsecond=0) - - # 计算前一年到上月的时间范围 - one_year_ago = current_time - pd.DateOffset(years=1, days=current_time.day - 1) - last_month = current_time - pd.DateOffset(months=1, days=current_time.day - 1) - - # 创建一个包含所有可能月份的日期范围 - all_months = pd.date_range(start=one_year_ago, end=last_month, freq="MS") - - # 获取每个城市-区域组合缺失的月份 - missing_months = [] - for (city, region), group in df.groupby(["城市", "区域"]): - available_months = group["月份"] - missing = all_months.difference(available_months) - if len(missing) > 0: - missing_str = ",".join([m.strftime("%Y%m") for m in missing]) - missing_months.append({"城市": city, "区域": region, "缺失月份": missing_str}) - - if missing_months: - # 创建DataFrame来存储结果 - missing_df = pd.DataFrame(missing_months) - path = 'E:\\goojia\\file\\excel文件\\airflow\\missing_region.xlsx' - missing_df.to_excel(path, index=False) - - # 如果区域在 missing_df 中,则将 df 中的该区域数据删除 - for city, region in zip(missing_df['城市'], missing_df['区域']): - df = df[(df['城市'] != city) | (df['区域'] != region)] - - df['月份'] = df['月份1'] - # 删除月份1列 - df.drop(columns=['月份1'], inplace=True) - output_path = 'E:\\goojia\\file\\excel文件\\airflow\\all_region_price_with_change.xlsx' - df.to_excel(output_path, index=False) - logging.info(f"区域涨跌幅数据已写入Excel文件:{output_path}") - -region_missing() diff --git a/dags/stat/dag_increase_decrease_city_region.py b/dags/stat/dag_increase_decrease_city_region.py index b8e822a..6f9fe6f 100644 --- a/dags/stat/dag_increase_decrease_city_region.py +++ b/dags/stat/dag_increase_decrease_city_region.py @@ -1,17 +1,17 @@ -from airflow.operators.bash import BashOperator -import pendulum -from airflow import DAG - -with DAG( - dag_id="increase_decrease_city_region_dag", - # 每月的最后一天运行 - schedule="0 10 L * *", - start_date=pendulum.datetime(2024, 5, 30, tz="Asia/Shanghai"), - description="城市区域涨跌幅数据统计", - catchup=True -) as dag: - t1 = BashOperator( - task_id='increase_decrease_city_region', - bash_command='python /opt/airflow/dags/repo/dags/stat/cityPrice/increase_decrease_city_region.py', - ) - t1 +# from airflow.operators.bash import BashOperator +# import pendulum +# from airflow import DAG +# +# with DAG( +# dag_id="increase_decrease_city_region_dag", +# # 每月的最后一天运行 +# schedule="0 10 L * *", +# start_date=pendulum.datetime(2024, 5, 30, tz="Asia/Shanghai"), +# description="城市区域涨跌幅数据统计", +# catchup=True +# ) as dag: +# t1 = BashOperator( +# task_id='increase_decrease_city_region', +# bash_command='python /opt/airflow/dags/repo/dags/stat/cityPrice/increase_decrease_city_region.py', +# ) +# t1 diff --git a/dags/test/fail_test.py b/dags/test/fail_test.py deleted file mode 100644 index 3842f3f..0000000 --- a/dags/test/fail_test.py +++ /dev/null @@ -1,36 +0,0 @@ -from airflow import DAG -from airflow.operators.python_operator import PythonOperator -from datetime import datetime -from goojia_utils import dingutils - -# def send_dingding_alert(context): -# print('监测到错误') -# task_instance = context.get('task_instance') -# url = task_instance.log_url.replace("http://localhost:8080", "https://airflow.fangjia.com") -# message = f"【airflow】 任务{task_instance.task_id}错误,请点击{url}查看!" -# print(message) -# dingutils.send_text(message, "ae14f50045e9375c354940a2afb86f14e453bd1f08f78da0ba6e788d35bccec4") -# print("success") -# -def fail_task(): - raise ValueError("This task is designed to fail") -# -# default_args = { -# "owner": "airflow", -# "retries": 0, -# "on_failure_callback": send_dingding_alert, # 任务失败时触发钉钉告警 -# } - - -dag = DAG( - 'test_failure_notification', - #default_args=default_args, - start_date=datetime(2025, 4, 22), - schedule=None -) - -task = PythonOperator( - task_id='failing_task', - python_callable=fail_task, - dag=dag -) \ No newline at end of file