From 8a8a9363e0596cce0056a981a0e1571b918fc2b4 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Tue, 15 Nov 2022 10:07:23 +0800 Subject: [PATCH] =?UTF-8?q?=E8=9E=8D=E5=90=88=E6=AF=8F=E6=97=A5=E6=A3=80?= =?UTF-8?q?=E6=B5=8B=E8=BF=81=E7=A7=BB=E8=87=B3airflow?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/jh/merge/merge.py | 355 +++++++++++++++++++++++++++++++++++++++++ dags/jh/merge_dag.py | 17 ++ 2 files changed, 372 insertions(+) create mode 100644 dags/jh/merge/merge.py create mode 100644 dags/jh/merge_dag.py diff --git a/dags/jh/merge/merge.py b/dags/jh/merge/merge.py new file mode 100644 index 0000000..07ee54b --- /dev/null +++ b/dags/jh/merge/merge.py @@ -0,0 +1,355 @@ +#!/usr/bin/env python +# coding: utf-8 + +# In[56]: + + +import os +import tarfile +from goojia_utils import dateutils, ftputils, tarutils +import datetime +import calendar +import requests +import json +import shutil + + +# In[9]: + + +import ftplib +import os +import socket +import stat +from os.path import isdir + +from numpy import unicode + +import paramiko + + +def _check_local(local): + if not os.path.exists(local): + try: + os.mkdir(local) + except IOError as err: + print(err) + + +class SftpClient: + def __init__(self): + self.client = None + self.sftp = None + pass + + def sftp_connect(self, username, password, host, port=22): + """ + 建立连接,获取sftp句柄 + :param username + :param password + :param host + :param port + :return: + """ + try: + self.client = paramiko.Transport((host, port)) + except Exception as error: + print(error) + else: + try: + self.client.connect(username=username, password=password) + except Exception as error: + print(error) + else: + self.sftp = paramiko.SFTPClient.from_transport(self.client) + + def get(self, remote, local): + """ + 从远程机器上下载文件(夹)到本地 + :param remote 远程路径 + :param local 本地路径 + """ + try: + result = self.sftp.stat(remote) + except IOError as err: + error = '[ERROR %s] %s: %s' % (err.errno, os.path.basename(os.path.normpath(remote)), err.strerror) + print(error) + else: + # 判断远程文件是否为目录 + if stat.S_ISDIR(result.st_mode): + dirname = os.path.basename(os.path.normpath(remote)) + local = os.path.join(local, dirname) + _check_local(local) + for file in self.sftp.listdir(remote): + sub_remote = os.path.join(remote, file) + sub_remote = sub_remote.replace('\\', '/') + self.get(sub_remote, local) + else: + # 拷贝文件 + if os.path.isdir(local): + local = os.path.join(local, os.path.basename(remote)) + try: + self.sftp.get(remote, local) + except IOError as err: + print(err) + else: + print('[get]', local, '<==', remote) + + # 断开连接 + def disconnect(self): + try: + self.client.close() + except Exception as error: + print(error) + error + + +class FtpClient: + + def __init__(self): + self.ftp = ftplib.FTP() + self.ftp.set_debuglevel(2) + pass + + def connect_ftp(self, host, port, timeout=60, user=None, passwd=None): + """ + 连接登录ftp + :param host: + :param port: + :param user: + :param passwd: + :return: + """ + try: + self.ftp.connect(host=host, port=port, timeout=timeout) + print("*******已经成功连接服务器FTP服务!!!") + except(socket.error, socket.gaierror) as e: + print("错误:无法访问服务器FTP服务!!!") + print(e) + exit() + try: + self.ftp.login(user=user, passwd=passwd) + except ftplib.error_perm as e: + print("登录失败,请检查用户名和密码,错误代码") + print(e) + exit() + + def cat_dir(self, filepath): + """ + 显示ftp指定目录下的文件 + :param filepath: + :return: + """ + return self.ftp.dir(filepath) + + def make_dir(self, filepath): + """ + 新建ftp下的目录,ftp目录下不存在此文件 + :param filepath: + :return: + """ + try: + filepath.encode("utf-8") + mkd = self.ftp.mkd(filepath) + return mkd + except ftplib.error_perm as e: + print("请查看目录下是否存在同名文件,异常为") + print(e) + + def remove_dir(self, dirname): + """ + 删除目录 + :param dirname + """ + try: + self.ftp.rmd(dirname) + except ftplib.error_perm as e: + print("异常:!!!目录已经被删除了") + print(e) + + def download_file(self, localpath, remotepath, filename): + """ + 从FTP指定目录上下载文件,并保存到本地指定路径 + :param localpath: linux上路径 + :param remotepath: 本地路径 + :param filename: 文件名 + """ + bufsize = 8192 # 设置缓冲区大小; + ulocalpath = unicode(localpath, "utf-8") + try: + self.ftp.cwd(remotepath) + filepath = ulocalpath + '/' + file_handler = open(filepath + filename, "wb").write # 以写模式打开本地文件,---需下载到本地的路径 + # 传一个回调函数给retrbinary() 它在每接收一个二进制数据时都会被调用 + self.ftp.retrbinary("RETR " % filename, file_handler, bufsize) # 接收服务器上文件并写入本地文件 + except ftplib.error_perm as e: + print("请检查路径是否正确,是否有权限读取,错误代码") + print(e) + + def upload(self, localpath): + """ + 上传至ftp + :param localpath: linux上路径 + """ + bufsize = 8192 + ulocalpath = unicode(localpath, "utf-8") + try: + file_handler = open(ulocalpath, "rb") + filename = os.path.split(ulocalpath)[-1] + try: + self.ftp.storbinary("stor " % filename, file_handler, bufsize) + except ftplib.error_perm as e: + print("请检查上传路径是否正确,是否连接FTP,---错误代码") + print(e) + except e: + print("请检查本地路径是否存在····") + + def lst_file(self, path): + """ + 获取指定目录下文件 + :param path + """ + s = [] + try: + for name in self.ftp.nlst(path): + name1 = unicode(name, "utf-8") + s.append(name1) + return s + except ftplib.all_errors as e: + print("请检查目录是否正确,错误代码时-----") + print(e) + + def close(self): + """ + 退出ftp + """ + self.ftp.quit() + +result = [] + +# #### 检测传输压缩文件是否完整 + +# ##### 1. 融合每天检测 + +try: + today = dateutils.add_months(datetime.datetime.now(), days=1) + date = dateutils.format_date(today, '%Y%m%d') + print('当前月:' + date) + curr_month_days = calendar.monthrange(today.year, today.month)[1] + print('当月天数:' + str(curr_month_days)) + + + # In[79]: + + + path = '/home/airflow/data/rh' + if not os.path.exists(path): + os.makedirs(path) + + client = SftpClient() + client.sftp_connect(username="ccb", password="ccb!@#", host="192.168.0.45", port=17495) + client.get(remote="/home/ccb_rh/output/" + date, local=path) + client.disconnect() + + + # In[80]: + + + cityids=['440400','210100','469028','330100','441900','440500','370300','320100','530100','140200','370200', + '321000','640100','220100','350500','520300','330600','371100','360400','340800','410300','330200', + '331000','510100','321100','540100','630100','441200','430100','330500','130900','441800','469005', + '620100','130200','610100','330400','340100','440800','440100','131000','321200','350600','440300', + '370600','441300','110000','420500','320900','500000','340200','320500','350100','310000','230100', + '150100','220200','330300','469029','410100','650100','420100','442000','120000','320200','460200', + '130100','450100','450500','430700','370100','520100','450200','320800','320400','320300','320600', + '140100','210700','130700','350200','430600','320700','000000','440600','430200','130600','360100', + '330700','130300','340500','460100','210200'] + len(cityids) + + +# In[81]: + + + dirs = os.listdir(path + '/' + date) + if date + '.ok' not in dirs: + result.append(date + '.ok' + '文件不存在') + # 每月都要生成 + if date.endswith('10') == False: + name = '000000_' + date + '_FJW' + if name + '.tar.gz' not in dirs: + result.append(name + '.tar.gz' + '文件不存在') + tarutils.untar_file(path + '/' + date + "/" + name + '.tar.gz', path + '/' + date + '/' + name) + sub_dirs = os.listdir(path + '/' + date + '/' + name) + if 'FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_' + date + '_0001.dat' not in sub_dirs: + result.append('FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_' + date + '_0001.dat文件不存在') + else: + cityid = '000000' + name = cityid + '_' + date + '_FJW' + if name + '.tar.gz' not in dirs: + result.append(name + '.tar.gz' + '文件不存在') + tarutils.untar_file(path + '/' + date + "/" + name + '.tar.gz', path + '/' + date + '/' + name) + # 全国 + sub_dirs = os.listdir(path + '/' + date + '/' + name) + if 'FJW_000000_' + cityid + '_EVALUATE_RATE_FJW_ALL_' + date + '_0001.dat' not in sub_dirs: + result.append('FJW_000000_' + cityid + '_EVALUATE_RATE_FJW_ALL_' + date + '_0001.dat文件不存在') + + ## 固定13个文件 + if 'FJW_000000_' + cityid + '_ADDRESS_ALIAS_FJW_ALL_' + date + '_0001.dat' not in sub_dirs: + result.append('FJW_000000_' + cityid + '_ADDRESS_ALIAS_FJW_ALL_' + date + '_0001.dat文件不存在') + if 'FJW_000000_' + cityid + '_BIGDATA_BUILDING_FJW_ALL_' + date + '_0001.dat' not in sub_dirs: + result.append('FJW_000000_' + cityid + '_BIGDATA_BUILDING_FJW_ALL_' + date + '_0001.dat文件不存在') + if 'FJW_000000_' + cityid + '_BIGDATA_BUILDPROJ_FJW_ALL_' + date + '_0001.dat' not in sub_dirs: + result.append('FJW_000000_' + cityid + '_BIGDATA_BUILDPROJ_FJW_ALL_' + date + '_0001.dat文件不存在') + if 'FJW_000000_' + cityid + '_BIGDATA_COMMBUILDSTRU_FJW_ALL_' + date + '_0001.dat' not in sub_dirs: + result.append('FJW_000000_' + cityid + '_BIGDATA_COMMBUILDSTRU_FJW_ALL_' + date + '_0001.dat文件不存在') + if 'FJW_000000_' + cityid + '_BIGDATA_COMMBUILDYEAR_FJW_ALL_' + date + '_0001.dat' not in sub_dirs: + result.append('FJW_000000_' + cityid + '_BIGDATA_COMMBUILDYEAR_FJW_ALL_' + date + '_0001.dat文件不存在') + + if 'FJW_000000_' + cityid + '_BIGDATA_COMMFEE_FJW_ALL_' + date + '_0001.dat' not in sub_dirs: + result.append('FJW_000000_' + cityid + '_BIGDATA_COMMFEE_FJW_ALL_' + date + '_0001.dat文件不存在') + if 'FJW_000000_' + cityid + '_BIGDATA_COMMHOUSETYPE_FJW_ALL_' + date + '_0001.dat' not in sub_dirs: + result.append('FJW_000000_' + cityid + '_BIGDATA_COMMHOUSETYPE_FJW_ALL_' + date + '_0001.dat文件不存在') + if 'FJW_000000_' + cityid + '_BIGDATA_COMMHOUSEUSE_FJW_ALL_' + date + '_0001.dat' not in sub_dirs: + result.append('FJW_000000_' + cityid + '_BIGDATA_COMMHOUSEUSE_FJW_ALL_' + date + '_0001.dat文件不存在') + if 'FJW_000000_' + cityid + '_BIGDATA_COMMLAYOUTPIC_FJW_ALL_' + date + '_0001.dat' not in sub_dirs: + result.append('FJW_000000_' + cityid + '_BIGDATA_COMMLAYOUTPIC_FJW_ALL_' + date + '_0001.dat文件不存在') + if 'FJW_000000_' + cityid + '_BIGDATA_COMMUNITY_FJW_ALL_' + date + '_0001.dat' not in sub_dirs: + result.append('FJW_000000_' + cityid + '_BIGDATA_COMMUNITY_FJW_ALL_' + date + '_0001.dat文件不存在') + + if 'FJW_000000_' + cityid + '_BIGDATA_DISTRICT_FJW_ALL_' + date + '_0001.dat' not in sub_dirs: + result.append('FJW_000000_' + cityid + '_BIGDATA_DISTRICT_FJW_ALL_' + date + '_0001.dat文件不存在') + if 'FJW_000000_' + cityid + '_BIGDATA_POSITION_FJW_ALL_' + date + '_0001.dat' not in sub_dirs: + result.append('FJW_000000_' + cityid + '_BIGDATA_POSITION_FJW_ALL_' + date + '_0001.dat文件不存在') + if 'FJW_000000_' + cityid + '_BIGDATA_DEAL_FJW_ADD_' + date + '_0001.dat' not in sub_dirs: + result.append('FJW_000000_' + cityid + '_BIGDATA_DEAL_FJW_ADD_' + date + '_0001.dat文件不存在') + if len(result) == 0: + result.append('融合检测完毕未发现异常文件') + shutil.rmtree(path + "/" + date) + +except Exception as ex: + result.append(str(ex)) + + +# In[82]: + + +print(result) + + + +# In[76]: + + +infos = "【" + date + "自动检测】" +for item in result: + infos += item + "\n" +data = { + "msgtype": "text", + "text": { + "content": infos + } +} +# 发送消息至群 +requests.post(url='https://oapi.dingtalk.com/robot/send?access_token=1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736', + json=data) + diff --git a/dags/jh/merge_dag.py b/dags/jh/merge_dag.py new file mode 100644 index 0000000..b497582 --- /dev/null +++ b/dags/jh/merge_dag.py @@ -0,0 +1,17 @@ +from airflow.operators.bash import BashOperator +import pendulum +from airflow import DAG + +with DAG( + dag_id="merge_dag", + # 每日下午4点 + schedule_interval="0 16 * * *", + start_date=pendulum.datetime(2022, 8, 23, tz="Asia/Shanghai"), + catchup=True, + description="每日融合文件检测" +) as dag: + t1 = BashOperator( + task_id='merge', + bash_command='python /opt/airflow/dags/repo/dags/jh/merge/merge.py', + ) + t1