This commit is contained in:
沈健
2024-07-22 17:37:38 +08:00
parent 89c472d131
commit 202a5b9198
2 changed files with 51 additions and 3 deletions
+16
View File
@@ -0,0 +1,16 @@
from airflow.operators.bash import BashOperator
import pendulum
from airflow import DAG
with DAG(
dag_id="third_gxd_dag",
#schedule_interval="30 16 27 * *",
start_date=pendulum.datetime(2024, 7, 22, tz="Asia/Shanghai"),
catchup=True,
description="第三方均价及国信达价格输出"
) as dag:
t1 = BashOperator(
task_id='gxd_dag',
bash_command='python /opt/airflow/dags/repo/dags/jh/price/price_third_gxd.py {{dag_run.conf["yearmonth"] if "yearmonth" in dag_run.conf}} ',
)
t1
+35 -3
View File
@@ -1,9 +1,21 @@
import logging
import os
import sys
import pandas as pd
import requests
from goojia_utils import dateutils, nextcloudutils
from dags.configs.citymap import city_name_id_map
from pqsql_connect import get_community_base_price,get_evaluation_third_price,get_third_price
from mongodb_connect import get_district_stat_on,get_seaweed,get_gxd_price
yearmonth='202404'
latest_year_month = '202403'
yearmonth = sys.argv[1] if len(sys.argv) > 1 else dateutils.format_date(pd.datetime.utcnow(), "%Y%m")
latest_year_month = dateutils.format_date(dateutils.add_months(dateutils.parse_date(yearmonth, '%Y%m'), months=-1), '%Y%m')
pathAll = "/home/airflow/data/third_gxd"
if not os.path.exists(pathAll):
os.makedirs(pathAll)
df_concat = pd.DataFrame()
for city in city_name_id_map:
print(city)
@@ -44,6 +56,26 @@ for city in city_name_id_map:
price_merge = price_merge.merge(data_df_temp[['_id','国信达价格']], on='_id', how='left')
price_merge.rename(columns={'city': '城市', 'district': '区域', '_id': '小区ID', 'name': '小区名','yearmonth':'月份'}, inplace=True)
df_concat = pd.concat([df_concat,price_merge])
df_concat.to_excel('小区第三方均价输出.xlsx', encoding='utf-8', index=False)
df_concat.to_excel(yearmonth + '小区第三方均价输出.xlsx', encoding='utf-8', index=False)
output_path = pathAll + "/" + yearmonth + '小区第三方均价输出.xlsx'
df_concat.to_excel(output_path, index=False)
cloudPath = '共享/统计/' + yearmonth + '小区第三方均价输出.xlsx'
nextcloudutils.up_to_nextcloud(output_path, cloudPath)
logging.info(f"区域涨跌幅数据Excel文件已经上传到云盘")
infos = "【第三方价格自动检测】"
data = {
"msgtype": "text",
"text": {
"content": infos + "数据Excel文件已经上传到云盘, 请点击 https://nextcloud.fangjia.com/s/3aw663fy9BbiBeL 查看详情"
}
}
# 发送消息至群
requests.post(url='https://oapi.dingtalk.com/robot/send?access_token=1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736',
json=data)