优化估值检测
This commit is contained in:
@@ -0,0 +1,16 @@
|
||||
from airflow.operators.bash import BashOperator
|
||||
import pendulum
|
||||
from airflow import DAG
|
||||
|
||||
with DAG(
|
||||
dag_id="evaluate_dag",
|
||||
schedule_interval="30 16 27 * *",
|
||||
start_date=pendulum.datetime(2023, 3, 28, tz="Asia/Shanghai"),
|
||||
catchup=True,
|
||||
description="每月27日估值文件检测"
|
||||
) as dag:
|
||||
t1 = BashOperator(
|
||||
task_id='merge',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/jh/merge/evaluate.py {{dag_run.conf["day"] if "day" in dag_run.conf}}',
|
||||
)
|
||||
t1
|
||||
@@ -0,0 +1,131 @@
|
||||
#!/usr/bin/env python
|
||||
# coding: utf-8
|
||||
|
||||
from goojia_utils import dateutils, ftputils, tarutils
|
||||
import datetime
|
||||
import shutil
|
||||
import os
|
||||
import requests
|
||||
import logging
|
||||
|
||||
# #### 检测传输压缩文件是否完整
|
||||
|
||||
# ##### 2.估值每月27日检测
|
||||
|
||||
# In[60]:
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
date = dateutils.format_date(datetime.datetime.now(), '%Y%m') + '28'
|
||||
|
||||
path = '/home/airflow/data/gz'
|
||||
local_aim_dir = path + '/' + date
|
||||
if not os.path.exists(path):
|
||||
os.makedirs(path)
|
||||
if not os.path.exists(local_aim_dir):
|
||||
os.makedirs(local_aim_dir)
|
||||
|
||||
|
||||
client = ftputils.SftpClient()
|
||||
client.sftp_connect(username="ccb", password="ccb!@#", host="192.168.0.45", port=17495)
|
||||
client.get(remote="/home/ccb_rh/output/" + date + '/WS_GJ_000000_' + date + '_0001.tar', local=local_aim_dir)
|
||||
client.get(remote="/home/ccb_rh/output/" + date + '/' + date + '.ok', local=path + '/' + date)
|
||||
client.get(remote="/home/ccb_rh/output/" + date + '/WS_GJ_000000_' + date + '_0002.tar', local=local_aim_dir)
|
||||
client.disconnect()
|
||||
|
||||
|
||||
# In[57]:
|
||||
|
||||
|
||||
## 解压
|
||||
result = []
|
||||
try:
|
||||
dirs = os.listdir(local_aim_dir)
|
||||
if date + '.ok' not in dirs:
|
||||
result.append(date + '.ok' + '文件不存在')
|
||||
tar_file = 'WS_GJ_000000_' + date + '_0001.tar'
|
||||
has_two_tar = False
|
||||
if 'WS_GJ_000000_' + date + '_0002.tar' in dirs:
|
||||
has_two_tar = True
|
||||
shutil.rmtree(local_aim_dir + '/' + date + '_2')
|
||||
tarutils.untar_file(local_aim_dir + '/WS_GJ_000000_' + date + '_0002.tar', local_aim_dir + '/')
|
||||
if tar_file not in dirs:
|
||||
result.append(tar_file + '文件不存在')
|
||||
else:
|
||||
if has_two_tar is True:
|
||||
shutil.rmtree(local_aim_dir + '/' + date + '_1')
|
||||
tarutils.untar_file(local_aim_dir + "/" + tar_file, local_aim_dir + '/')
|
||||
if has_two_tar is False:
|
||||
sub_dirs = os.listdir(local_aim_dir)
|
||||
else:
|
||||
sub_dirs = os.listdir(local_aim_dir + '/' + date + '_1')
|
||||
sub_dirs.extend(os.listdir(local_aim_dir + '/' + date + '_2'))
|
||||
if 'MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_' + date + '_0001.dat文件不存在')
|
||||
if 'MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_' + date + '_0001.dat文件不存在')
|
||||
if 'MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_' + date + '_0001.dat文件不存在')
|
||||
if 'MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_' + date + '_0001.dat文件不存在')
|
||||
if 'MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_' + date + '_0001.dat文件不存在')
|
||||
if 'MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_' + date + '_0001.dat文件不存在')
|
||||
if 'MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_' + date + '_0001.dat文件不存在')
|
||||
if 'MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_' + date + '_0001.dat文件不存在')
|
||||
if 'MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_' + date + '_0001.dat文件不存在')
|
||||
if 'MODEL_000000_000000_BIGDATA_POI_GJ_ALL_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('MODEL_000000_000000_BIGDATA_POI_GJ_ALL_' + date + '_0001.dat文件不存在')
|
||||
|
||||
if 'MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_' + date + '_0001.dat文件不存在')
|
||||
if 'MODEL_000000_000000_CITY_PRICE_GJ_ALL_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('MODEL_000000_000000_CITY_PRICE_GJ_ALL_' + date + '_0001.dat文件不存在')
|
||||
if 'MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_' + date + '_0001.dat文件不存在')
|
||||
if 'MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_' + date + '_0001.dat文件不存在')
|
||||
if 'MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_' + date + '_0001.dat文件不存在')
|
||||
if 'MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_' + date + '_0001.dat文件不存在')
|
||||
if 'MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_' + date + '_0001.dat文件不存在')
|
||||
if 'MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_' + date + '_0001.dat文件不存在')
|
||||
if 'MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_' + date + '_0001.dat文件不存在')
|
||||
if 'MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_' + date + '_0001.dat文件不存在')
|
||||
|
||||
if 'MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_' + date + '_0001.dat文件不存在')
|
||||
|
||||
if len(result) == 0:
|
||||
result.append('估值检测完毕未发现异常文件')
|
||||
shutil.rmtree(local_aim_dir)
|
||||
else:
|
||||
result.append(" @高娟 ")
|
||||
|
||||
except Exception as ex:
|
||||
result.append(" @高娟 ")
|
||||
result.append(str(ex))
|
||||
|
||||
infos = "【" + date + "自动检测】"
|
||||
for item in result:
|
||||
infos += item + "\n"
|
||||
data = {
|
||||
"msgtype": "text",
|
||||
"text": {
|
||||
"content": infos
|
||||
}
|
||||
}
|
||||
log.info(data)
|
||||
|
||||
# 发送消息至群
|
||||
# requests.post(url='https://oapi.dingtalk.com/robot/send?access_token=1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736',
|
||||
# json=data)
|
||||
|
||||
+5
-10
@@ -4,7 +4,9 @@ import requests
|
||||
import shutil
|
||||
import os
|
||||
import sys
|
||||
import logging
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
result = []
|
||||
|
||||
# ##### 1. 融合每天检测
|
||||
@@ -132,22 +134,14 @@ try:
|
||||
if len(result) == 0:
|
||||
result.append('融合检测完毕未发现异常文件')
|
||||
shutil.rmtree(path + "/" + date)
|
||||
else:
|
||||
result.append(" @高娟 ")
|
||||
|
||||
except Exception as ex:
|
||||
result.append(" @高娟 ")
|
||||
result.append(str(ex))
|
||||
|
||||
|
||||
# In[82]:
|
||||
|
||||
|
||||
print(result)
|
||||
|
||||
|
||||
|
||||
# In[76]:
|
||||
|
||||
|
||||
infos = "【" + date + "自动检测】"
|
||||
for item in result:
|
||||
infos += item + "\n"
|
||||
@@ -157,6 +151,7 @@ data = {
|
||||
"content": infos
|
||||
}
|
||||
}
|
||||
log.info(data)
|
||||
# 发送消息至群
|
||||
requests.post(url='https://oapi.dingtalk.com/robot/send?access_token=1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736',
|
||||
json=data)
|
||||
|
||||
Reference in New Issue
Block a user