新增监控四张表

This commit is contained in:
shenjian@fangjia.com
2023-02-03 10:03:30 +08:00
parent 92c2b6dfa8
commit aeaa132b4e
+9 -215
View File
@@ -4,8 +4,6 @@
# In[56]:
import os
import tarfile
from goojia_utils import dateutils, ftputils, tarutils
import datetime
import calendar
@@ -13,217 +11,7 @@ import requests
import json
import shutil
# In[9]:
import ftplib
import os
import socket
import stat
from os.path import isdir
from numpy import unicode
import paramiko
def _check_local(local):
if not os.path.exists(local):
try:
os.mkdir(local)
except IOError as err:
print(err)
class SftpClient:
def __init__(self):
self.client = None
self.sftp = None
pass
def sftp_connect(self, username, password, host, port=22):
"""
建立连接,获取sftp句柄
:param username
:param password
:param host
:param port
:return:
"""
try:
self.client = paramiko.Transport((host, port))
except Exception as error:
print(error)
else:
try:
self.client.connect(username=username, password=password)
except Exception as error:
print(error)
else:
self.sftp = paramiko.SFTPClient.from_transport(self.client)
def get(self, remote, local):
"""
从远程机器上下载文件(夹)到本地
:param remote 远程路径
:param local 本地路径
"""
try:
result = self.sftp.stat(remote)
except IOError as err:
error = '[ERROR %s] %s: %s' % (err.errno, os.path.basename(os.path.normpath(remote)), err.strerror)
print(error)
else:
# 判断远程文件是否为目录
if stat.S_ISDIR(result.st_mode):
dirname = os.path.basename(os.path.normpath(remote))
local = os.path.join(local, dirname)
_check_local(local)
for file in self.sftp.listdir(remote):
sub_remote = os.path.join(remote, file)
sub_remote = sub_remote.replace('\\', '/')
self.get(sub_remote, local)
else:
# 拷贝文件
if os.path.isdir(local):
local = os.path.join(local, os.path.basename(remote))
try:
self.sftp.get(remote, local)
except IOError as err:
print(err)
else:
print('[get]', local, '<==', remote)
# 断开连接
def disconnect(self):
try:
self.client.close()
except Exception as error:
print(error)
error
class FtpClient:
def __init__(self):
self.ftp = ftplib.FTP()
self.ftp.set_debuglevel(2)
pass
def connect_ftp(self, host, port, timeout=60, user=None, passwd=None):
"""
连接登录ftp
:param host:
:param port:
:param user:
:param passwd:
:return:
"""
try:
self.ftp.connect(host=host, port=port, timeout=timeout)
print("*******已经成功连接服务器FTP服务!!!")
except(socket.error, socket.gaierror) as e:
print("错误:无法访问服务器FTP服务!!!")
print(e)
exit()
try:
self.ftp.login(user=user, passwd=passwd)
except ftplib.error_perm as e:
print("登录失败,请检查用户名和密码,错误代码")
print(e)
exit()
def cat_dir(self, filepath):
"""
显示ftp指定目录下的文件
:param filepath:
:return:
"""
return self.ftp.dir(filepath)
def make_dir(self, filepath):
"""
新建ftp下的目录,ftp目录下不存在此文件
:param filepath:
:return:
"""
try:
filepath.encode("utf-8")
mkd = self.ftp.mkd(filepath)
return mkd
except ftplib.error_perm as e:
print("请查看目录下是否存在同名文件,异常为")
print(e)
def remove_dir(self, dirname):
"""
删除目录
:param dirname
"""
try:
self.ftp.rmd(dirname)
except ftplib.error_perm as e:
print("异常:!!!目录已经被删除了")
print(e)
def download_file(self, localpath, remotepath, filename):
"""
从FTP指定目录上下载文件,并保存到本地指定路径
:param localpath: linux上路径
:param remotepath: 本地路径
:param filename: 文件名
"""
bufsize = 8192 # 设置缓冲区大小;
ulocalpath = unicode(localpath, "utf-8")
try:
self.ftp.cwd(remotepath)
filepath = ulocalpath + '/'
file_handler = open(filepath + filename, "wb").write # 以写模式打开本地文件,---需下载到本地的路径
# 传一个回调函数给retrbinary() 它在每接收一个二进制数据时都会被调用
self.ftp.retrbinary("RETR " % filename, file_handler, bufsize) # 接收服务器上文件并写入本地文件
except ftplib.error_perm as e:
print("请检查路径是否正确,是否有权限读取,错误代码")
print(e)
def upload(self, localpath):
"""
上传至ftp
:param localpath: linux上路径
"""
bufsize = 8192
ulocalpath = unicode(localpath, "utf-8")
try:
file_handler = open(ulocalpath, "rb")
filename = os.path.split(ulocalpath)[-1]
try:
self.ftp.storbinary("stor " % filename, file_handler, bufsize)
except ftplib.error_perm as e:
print("请检查上传路径是否正确,是否连接FTP,---错误代码")
print(e)
except e:
print("请检查本地路径是否存在····")
def lst_file(self, path):
"""
获取指定目录下文件
:param path
"""
s = []
try:
for name in self.ftp.nlst(path):
name1 = unicode(name, "utf-8")
s.append(name1)
return s
except ftplib.all_errors as e:
print("请检查目录是否正确,错误代码时-----")
print(e)
def close(self):
"""
退出ftp
"""
self.ftp.quit()
result = []
@@ -245,13 +33,18 @@ try:
path = '/home/airflow/data/rh'
if not os.path.exists(path):
os.makedirs(path)
if not os.path.exists(path + "/" + date):
os.makedirs(path + "/" + date)
client = SftpClient()
client = ftputils.SftpClient()
client.sftp_connect(username="ccb", password="ccb!@#", host="192.168.0.45", port=17495)
client.get(remote="/home/ccb_rh/output/" + date, local=path)
client.get(remote="/home/ccb_rh/output/" + date + '/000000_' + date + '_FJW.tar.gz', local=path + '/' + date)
client.get(remote="/home/ccb_rh/output/" + date + '/' + date + '.ok', local=path + '/' + date)
client.disconnect()
# In[80]:
@@ -328,7 +121,8 @@ try:
result.append('FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_' + date + '_0001.dat文件不存在')
if 'FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_' + date + '_0001.dat文件不存在')
if 'FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_' + date + '_0001.dat文件不存在')
if len(result) == 0:
result.append('融合检测完毕未发现异常文件')
shutil.rmtree(path + "/" + date)