From b5da624d933eaa0048b0ca8383cc68fda84d69a6 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Tue, 28 Mar 2023 14:53:40 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=BC=B0=E5=80=BC=E6=A3=80?= =?UTF-8?q?=E6=B5=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/jh/dag_evaluate.py | 16 +++++ dags/jh/merge/evaluation.py | 131 ++++++++++++++++++++++++++++++++++++ dags/jh/merge/merge.py | 15 ++--- 3 files changed, 152 insertions(+), 10 deletions(-) create mode 100644 dags/jh/dag_evaluate.py create mode 100644 dags/jh/merge/evaluation.py diff --git a/dags/jh/dag_evaluate.py b/dags/jh/dag_evaluate.py new file mode 100644 index 0000000..d40f28c --- /dev/null +++ b/dags/jh/dag_evaluate.py @@ -0,0 +1,16 @@ +from airflow.operators.bash import BashOperator +import pendulum +from airflow import DAG + +with DAG( + dag_id="evaluate_dag", + schedule_interval="30 16 27 * *", + start_date=pendulum.datetime(2023, 3, 28, tz="Asia/Shanghai"), + catchup=True, + description="每月27日估值文件检测" +) as dag: + t1 = BashOperator( + task_id='merge', + bash_command='python /opt/airflow/dags/repo/dags/jh/merge/evaluate.py {{dag_run.conf["day"] if "day" in dag_run.conf}}', + ) + t1 diff --git a/dags/jh/merge/evaluation.py b/dags/jh/merge/evaluation.py new file mode 100644 index 0000000..55e98b3 --- /dev/null +++ b/dags/jh/merge/evaluation.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python +# coding: utf-8 + +from goojia_utils import dateutils, ftputils, tarutils +import datetime +import shutil +import os +import requests +import logging + +# #### 检测传输压缩文件是否完整 + +# ##### 2.估值每月27日检测 + +# In[60]: + +log = logging.getLogger(__name__) +date = dateutils.format_date(datetime.datetime.now(), '%Y%m') + '28' + +path = '/home/airflow/data/gz' +local_aim_dir = path + '/' + date +if not os.path.exists(path): + os.makedirs(path) +if not os.path.exists(local_aim_dir): + os.makedirs(local_aim_dir) + + +client = ftputils.SftpClient() +client.sftp_connect(username="ccb", password="ccb!@#", host="192.168.0.45", port=17495) +client.get(remote="/home/ccb_rh/output/" + date + '/WS_GJ_000000_' + date + '_0001.tar', local=local_aim_dir) +client.get(remote="/home/ccb_rh/output/" + date + '/' + date + '.ok', local=path + '/' + date) +client.get(remote="/home/ccb_rh/output/" + date + '/WS_GJ_000000_' + date + '_0002.tar', local=local_aim_dir) +client.disconnect() + + +# In[57]: + + +## 解压 +result = [] +try: + dirs = os.listdir(local_aim_dir) + if date + '.ok' not in dirs: + result.append(date + '.ok' + '文件不存在') + tar_file = 'WS_GJ_000000_' + date + '_0001.tar' + has_two_tar = False + if 'WS_GJ_000000_' + date + '_0002.tar' in dirs: + has_two_tar = True + shutil.rmtree(local_aim_dir + '/' + date + '_2') + tarutils.untar_file(local_aim_dir + '/WS_GJ_000000_' + date + '_0002.tar', local_aim_dir + '/') + if tar_file not in dirs: + result.append(tar_file + '文件不存在') + else: + if has_two_tar is True: + shutil.rmtree(local_aim_dir + '/' + date + '_1') + tarutils.untar_file(local_aim_dir + "/" + tar_file, local_aim_dir + '/') + if has_two_tar is False: + sub_dirs = os.listdir(local_aim_dir) + else: + sub_dirs = os.listdir(local_aim_dir + '/' + date + '_1') + sub_dirs.extend(os.listdir(local_aim_dir + '/' + date + '_2')) + if 'MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_' + date + '_0001.dat' not in sub_dirs: + result.append('MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_' + date + '_0001.dat文件不存在') + if 'MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_' + date + '_0001.dat' not in sub_dirs: + result.append('MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_' + date + '_0001.dat文件不存在') + if 'MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_' + date + '_0001.dat' not in sub_dirs: + result.append('MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_' + date + '_0001.dat文件不存在') + if 'MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_' + date + '_0001.dat' not in sub_dirs: + result.append('MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_' + date + '_0001.dat文件不存在') + if 'MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_' + date + '_0001.dat' not in sub_dirs: + result.append('MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_' + date + '_0001.dat文件不存在') + if 'MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_' + date + '_0001.dat' not in sub_dirs: + result.append('MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_' + date + '_0001.dat文件不存在') + if 'MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_' + date + '_0001.dat' not in sub_dirs: + result.append('MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_' + date + '_0001.dat文件不存在') + if 'MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_' + date + '_0001.dat' not in sub_dirs: + result.append('MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_' + date + '_0001.dat文件不存在') + if 'MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_' + date + '_0001.dat' not in sub_dirs: + result.append('MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_' + date + '_0001.dat文件不存在') + if 'MODEL_000000_000000_BIGDATA_POI_GJ_ALL_' + date + '_0001.dat' not in sub_dirs: + result.append('MODEL_000000_000000_BIGDATA_POI_GJ_ALL_' + date + '_0001.dat文件不存在') + + if 'MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_' + date + '_0001.dat' not in sub_dirs: + result.append('MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_' + date + '_0001.dat文件不存在') + if 'MODEL_000000_000000_CITY_PRICE_GJ_ALL_' + date + '_0001.dat' not in sub_dirs: + result.append('MODEL_000000_000000_CITY_PRICE_GJ_ALL_' + date + '_0001.dat文件不存在') + if 'MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_' + date + '_0001.dat' not in sub_dirs: + result.append('MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_' + date + '_0001.dat文件不存在') + if 'MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_' + date + '_0001.dat' not in sub_dirs: + result.append('MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_' + date + '_0001.dat文件不存在') + if 'MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_' + date + '_0001.dat' not in sub_dirs: + result.append('MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_' + date + '_0001.dat文件不存在') + if 'MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_' + date + '_0001.dat' not in sub_dirs: + result.append('MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_' + date + '_0001.dat文件不存在') + if 'MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_' + date + '_0001.dat' not in sub_dirs: + result.append('MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_' + date + '_0001.dat文件不存在') + if 'MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_' + date + '_0001.dat' not in sub_dirs: + result.append('MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_' + date + '_0001.dat文件不存在') + if 'MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_' + date + '_0001.dat' not in sub_dirs: + result.append('MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_' + date + '_0001.dat文件不存在') + if 'MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_' + date + '_0001.dat' not in sub_dirs: + result.append('MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_' + date + '_0001.dat文件不存在') + + if 'MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_' + date + '_0001.dat' not in sub_dirs: + result.append('MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_' + date + '_0001.dat文件不存在') + + if len(result) == 0: + result.append('估值检测完毕未发现异常文件') + shutil.rmtree(local_aim_dir) + else: + result.append(" @高娟 ") + +except Exception as ex: + result.append(" @高娟 ") + result.append(str(ex)) + +infos = "【" + date + "自动检测】" +for item in result: + infos += item + "\n" +data = { + "msgtype": "text", + "text": { + "content": infos + } +} +log.info(data) + +# 发送消息至群 +# requests.post(url='https://oapi.dingtalk.com/robot/send?access_token=1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736', +# json=data) + diff --git a/dags/jh/merge/merge.py b/dags/jh/merge/merge.py index 929222e..60be1ba 100644 --- a/dags/jh/merge/merge.py +++ b/dags/jh/merge/merge.py @@ -4,7 +4,9 @@ import requests import shutil import os import sys +import logging +log = logging.getLogger(__name__) result = [] # ##### 1. 融合每天检测 @@ -132,22 +134,14 @@ try: if len(result) == 0: result.append('融合检测完毕未发现异常文件') shutil.rmtree(path + "/" + date) + else: + result.append(" @高娟 ") except Exception as ex: result.append(" @高娟 ") result.append(str(ex)) -# In[82]: - - -print(result) - - - -# In[76]: - - infos = "【" + date + "自动检测】" for item in result: infos += item + "\n" @@ -157,6 +151,7 @@ data = { "content": infos } } +log.info(data) # 发送消息至群 requests.post(url='https://oapi.dingtalk.com/robot/send?access_token=1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736', json=data)