From aeaa132b4e0b84bf43007bbe1b8890b808db7908 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Fri, 3 Feb 2023 10:03:30 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E7=9B=91=E6=8E=A7=E5=9B=9B?= =?UTF-8?q?=E5=BC=A0=E8=A1=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/jh/merge/merge.py | 224 ++--------------------------------------- 1 file changed, 9 insertions(+), 215 deletions(-) diff --git a/dags/jh/merge/merge.py b/dags/jh/merge/merge.py index 612422d..19f9483 100644 --- a/dags/jh/merge/merge.py +++ b/dags/jh/merge/merge.py @@ -4,8 +4,6 @@ # In[56]: -import os -import tarfile from goojia_utils import dateutils, ftputils, tarutils import datetime import calendar @@ -13,217 +11,7 @@ import requests import json import shutil - -# In[9]: - - -import ftplib import os -import socket -import stat -from os.path import isdir - -from numpy import unicode - -import paramiko - - -def _check_local(local): - if not os.path.exists(local): - try: - os.mkdir(local) - except IOError as err: - print(err) - - -class SftpClient: - def __init__(self): - self.client = None - self.sftp = None - pass - - def sftp_connect(self, username, password, host, port=22): - """ - 建立连接,获取sftp句柄 - :param username - :param password - :param host - :param port - :return: - """ - try: - self.client = paramiko.Transport((host, port)) - except Exception as error: - print(error) - else: - try: - self.client.connect(username=username, password=password) - except Exception as error: - print(error) - else: - self.sftp = paramiko.SFTPClient.from_transport(self.client) - - def get(self, remote, local): - """ - 从远程机器上下载文件(夹)到本地 - :param remote 远程路径 - :param local 本地路径 - """ - try: - result = self.sftp.stat(remote) - except IOError as err: - error = '[ERROR %s] %s: %s' % (err.errno, os.path.basename(os.path.normpath(remote)), err.strerror) - print(error) - else: - # 判断远程文件是否为目录 - if stat.S_ISDIR(result.st_mode): - dirname = os.path.basename(os.path.normpath(remote)) - local = os.path.join(local, dirname) - _check_local(local) - for file in self.sftp.listdir(remote): - sub_remote = os.path.join(remote, file) - sub_remote = sub_remote.replace('\\', '/') - self.get(sub_remote, local) - else: - # 拷贝文件 - if os.path.isdir(local): - local = os.path.join(local, os.path.basename(remote)) - try: - self.sftp.get(remote, local) - except IOError as err: - print(err) - else: - print('[get]', local, '<==', remote) - - # 断开连接 - def disconnect(self): - try: - self.client.close() - except Exception as error: - print(error) - error - - -class FtpClient: - - def __init__(self): - self.ftp = ftplib.FTP() - self.ftp.set_debuglevel(2) - pass - - def connect_ftp(self, host, port, timeout=60, user=None, passwd=None): - """ - 连接登录ftp - :param host: - :param port: - :param user: - :param passwd: - :return: - """ - try: - self.ftp.connect(host=host, port=port, timeout=timeout) - print("*******已经成功连接服务器FTP服务!!!") - except(socket.error, socket.gaierror) as e: - print("错误:无法访问服务器FTP服务!!!") - print(e) - exit() - try: - self.ftp.login(user=user, passwd=passwd) - except ftplib.error_perm as e: - print("登录失败,请检查用户名和密码,错误代码") - print(e) - exit() - - def cat_dir(self, filepath): - """ - 显示ftp指定目录下的文件 - :param filepath: - :return: - """ - return self.ftp.dir(filepath) - - def make_dir(self, filepath): - """ - 新建ftp下的目录,ftp目录下不存在此文件 - :param filepath: - :return: - """ - try: - filepath.encode("utf-8") - mkd = self.ftp.mkd(filepath) - return mkd - except ftplib.error_perm as e: - print("请查看目录下是否存在同名文件,异常为") - print(e) - - def remove_dir(self, dirname): - """ - 删除目录 - :param dirname - """ - try: - self.ftp.rmd(dirname) - except ftplib.error_perm as e: - print("异常:!!!目录已经被删除了") - print(e) - - def download_file(self, localpath, remotepath, filename): - """ - 从FTP指定目录上下载文件,并保存到本地指定路径 - :param localpath: linux上路径 - :param remotepath: 本地路径 - :param filename: 文件名 - """ - bufsize = 8192 # 设置缓冲区大小; - ulocalpath = unicode(localpath, "utf-8") - try: - self.ftp.cwd(remotepath) - filepath = ulocalpath + '/' - file_handler = open(filepath + filename, "wb").write # 以写模式打开本地文件,---需下载到本地的路径 - # 传一个回调函数给retrbinary() 它在每接收一个二进制数据时都会被调用 - self.ftp.retrbinary("RETR " % filename, file_handler, bufsize) # 接收服务器上文件并写入本地文件 - except ftplib.error_perm as e: - print("请检查路径是否正确,是否有权限读取,错误代码") - print(e) - - def upload(self, localpath): - """ - 上传至ftp - :param localpath: linux上路径 - """ - bufsize = 8192 - ulocalpath = unicode(localpath, "utf-8") - try: - file_handler = open(ulocalpath, "rb") - filename = os.path.split(ulocalpath)[-1] - try: - self.ftp.storbinary("stor " % filename, file_handler, bufsize) - except ftplib.error_perm as e: - print("请检查上传路径是否正确,是否连接FTP,---错误代码") - print(e) - except e: - print("请检查本地路径是否存在····") - - def lst_file(self, path): - """ - 获取指定目录下文件 - :param path - """ - s = [] - try: - for name in self.ftp.nlst(path): - name1 = unicode(name, "utf-8") - s.append(name1) - return s - except ftplib.all_errors as e: - print("请检查目录是否正确,错误代码时-----") - print(e) - - def close(self): - """ - 退出ftp - """ - self.ftp.quit() result = [] @@ -245,13 +33,18 @@ try: path = '/home/airflow/data/rh' if not os.path.exists(path): os.makedirs(path) + if not os.path.exists(path + "/" + date): + os.makedirs(path + "/" + date) - client = SftpClient() + client = ftputils.SftpClient() client.sftp_connect(username="ccb", password="ccb!@#", host="192.168.0.45", port=17495) - client.get(remote="/home/ccb_rh/output/" + date, local=path) + client.get(remote="/home/ccb_rh/output/" + date + '/000000_' + date + '_FJW.tar.gz', local=path + '/' + date) + client.get(remote="/home/ccb_rh/output/" + date + '/' + date + '.ok', local=path + '/' + date) client.disconnect() + + # In[80]: @@ -328,7 +121,8 @@ try: result.append('FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_' + date + '_0001.dat文件不存在') if 'FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_' + date + '_0001.dat' not in sub_dirs: result.append('FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_' + date + '_0001.dat文件不存在') - + if 'FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_' + date + '_0001.dat' not in sub_dirs: + result.append('FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_' + date + '_0001.dat文件不存在') if len(result) == 0: result.append('融合检测完毕未发现异常文件') shutil.rmtree(path + "/" + date)