优化
This commit is contained in:
@@ -0,0 +1,16 @@
|
||||
from airflow.operators.bash import BashOperator
|
||||
import pendulum
|
||||
from airflow import DAG
|
||||
|
||||
with DAG(
|
||||
dag_id="third_gxd_dag",
|
||||
#schedule_interval="30 16 27 * *",
|
||||
start_date=pendulum.datetime(2024, 7, 22, tz="Asia/Shanghai"),
|
||||
catchup=True,
|
||||
description="第三方均价及国信达价格输出"
|
||||
) as dag:
|
||||
t1 = BashOperator(
|
||||
task_id='gxd_dag',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/jh/price/price_third_gxd.py {{dag_run.conf["yearmonth"] if "yearmonth" in dag_run.conf}} ',
|
||||
)
|
||||
t1
|
||||
@@ -1,9 +1,21 @@
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
|
||||
import pandas as pd
|
||||
import requests
|
||||
from goojia_utils import dateutils, nextcloudutils
|
||||
|
||||
from dags.configs.citymap import city_name_id_map
|
||||
from pqsql_connect import get_community_base_price,get_evaluation_third_price,get_third_price
|
||||
from mongodb_connect import get_district_stat_on,get_seaweed,get_gxd_price
|
||||
yearmonth='202404'
|
||||
latest_year_month = '202403'
|
||||
yearmonth = sys.argv[1] if len(sys.argv) > 1 else dateutils.format_date(pd.datetime.utcnow(), "%Y%m")
|
||||
latest_year_month = dateutils.format_date(dateutils.add_months(dateutils.parse_date(yearmonth, '%Y%m'), months=-1), '%Y%m')
|
||||
|
||||
pathAll = "/home/airflow/data/third_gxd"
|
||||
if not os.path.exists(pathAll):
|
||||
os.makedirs(pathAll)
|
||||
|
||||
df_concat = pd.DataFrame()
|
||||
for city in city_name_id_map:
|
||||
print(city)
|
||||
@@ -44,6 +56,26 @@ for city in city_name_id_map:
|
||||
price_merge = price_merge.merge(data_df_temp[['_id','国信达价格']], on='_id', how='left')
|
||||
price_merge.rename(columns={'city': '城市', 'district': '区域', '_id': '小区ID', 'name': '小区名','yearmonth':'月份'}, inplace=True)
|
||||
df_concat = pd.concat([df_concat,price_merge])
|
||||
df_concat.to_excel('小区第三方均价输出.xlsx', encoding='utf-8', index=False)
|
||||
df_concat.to_excel(yearmonth + '小区第三方均价输出.xlsx', encoding='utf-8', index=False)
|
||||
|
||||
output_path = pathAll + "/" + yearmonth + '小区第三方均价输出.xlsx'
|
||||
df_concat.to_excel(output_path, index=False)
|
||||
|
||||
cloudPath = '共享/统计/' + yearmonth + '小区第三方均价输出.xlsx'
|
||||
nextcloudutils.up_to_nextcloud(output_path, cloudPath)
|
||||
logging.info(f"区域涨跌幅数据Excel文件已经上传到云盘")
|
||||
|
||||
|
||||
infos = "【第三方价格自动检测】"
|
||||
data = {
|
||||
"msgtype": "text",
|
||||
"text": {
|
||||
"content": infos + "数据Excel文件已经上传到云盘, 请点击 https://nextcloud.fangjia.com/s/3aw663fy9BbiBeL 查看详情"
|
||||
}
|
||||
}
|
||||
|
||||
# 发送消息至群
|
||||
requests.post(url='https://oapi.dingtalk.com/robot/send?access_token=1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736',
|
||||
json=data)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user