融合每日检测迁移至airflow
This commit is contained in:
@@ -0,0 +1,355 @@
|
||||
#!/usr/bin/env python
|
||||
# coding: utf-8
|
||||
|
||||
# In[56]:
|
||||
|
||||
|
||||
import os
|
||||
import tarfile
|
||||
from goojia_utils import dateutils, ftputils, tarutils
|
||||
import datetime
|
||||
import calendar
|
||||
import requests
|
||||
import json
|
||||
import shutil
|
||||
|
||||
|
||||
# In[9]:
|
||||
|
||||
|
||||
import ftplib
|
||||
import os
|
||||
import socket
|
||||
import stat
|
||||
from os.path import isdir
|
||||
|
||||
from numpy import unicode
|
||||
|
||||
import paramiko
|
||||
|
||||
|
||||
def _check_local(local):
|
||||
if not os.path.exists(local):
|
||||
try:
|
||||
os.mkdir(local)
|
||||
except IOError as err:
|
||||
print(err)
|
||||
|
||||
|
||||
class SftpClient:
|
||||
def __init__(self):
|
||||
self.client = None
|
||||
self.sftp = None
|
||||
pass
|
||||
|
||||
def sftp_connect(self, username, password, host, port=22):
|
||||
"""
|
||||
建立连接,获取sftp句柄
|
||||
:param username
|
||||
:param password
|
||||
:param host
|
||||
:param port
|
||||
:return:
|
||||
"""
|
||||
try:
|
||||
self.client = paramiko.Transport((host, port))
|
||||
except Exception as error:
|
||||
print(error)
|
||||
else:
|
||||
try:
|
||||
self.client.connect(username=username, password=password)
|
||||
except Exception as error:
|
||||
print(error)
|
||||
else:
|
||||
self.sftp = paramiko.SFTPClient.from_transport(self.client)
|
||||
|
||||
def get(self, remote, local):
|
||||
"""
|
||||
从远程机器上下载文件(夹)到本地
|
||||
:param remote 远程路径
|
||||
:param local 本地路径
|
||||
"""
|
||||
try:
|
||||
result = self.sftp.stat(remote)
|
||||
except IOError as err:
|
||||
error = '[ERROR %s] %s: %s' % (err.errno, os.path.basename(os.path.normpath(remote)), err.strerror)
|
||||
print(error)
|
||||
else:
|
||||
# 判断远程文件是否为目录
|
||||
if stat.S_ISDIR(result.st_mode):
|
||||
dirname = os.path.basename(os.path.normpath(remote))
|
||||
local = os.path.join(local, dirname)
|
||||
_check_local(local)
|
||||
for file in self.sftp.listdir(remote):
|
||||
sub_remote = os.path.join(remote, file)
|
||||
sub_remote = sub_remote.replace('\\', '/')
|
||||
self.get(sub_remote, local)
|
||||
else:
|
||||
# 拷贝文件
|
||||
if os.path.isdir(local):
|
||||
local = os.path.join(local, os.path.basename(remote))
|
||||
try:
|
||||
self.sftp.get(remote, local)
|
||||
except IOError as err:
|
||||
print(err)
|
||||
else:
|
||||
print('[get]', local, '<==', remote)
|
||||
|
||||
# 断开连接
|
||||
def disconnect(self):
|
||||
try:
|
||||
self.client.close()
|
||||
except Exception as error:
|
||||
print(error)
|
||||
error
|
||||
|
||||
|
||||
class FtpClient:
|
||||
|
||||
def __init__(self):
|
||||
self.ftp = ftplib.FTP()
|
||||
self.ftp.set_debuglevel(2)
|
||||
pass
|
||||
|
||||
def connect_ftp(self, host, port, timeout=60, user=None, passwd=None):
|
||||
"""
|
||||
连接登录ftp
|
||||
:param host:
|
||||
:param port:
|
||||
:param user:
|
||||
:param passwd:
|
||||
:return:
|
||||
"""
|
||||
try:
|
||||
self.ftp.connect(host=host, port=port, timeout=timeout)
|
||||
print("*******已经成功连接服务器FTP服务!!!")
|
||||
except(socket.error, socket.gaierror) as e:
|
||||
print("错误:无法访问服务器FTP服务!!!")
|
||||
print(e)
|
||||
exit()
|
||||
try:
|
||||
self.ftp.login(user=user, passwd=passwd)
|
||||
except ftplib.error_perm as e:
|
||||
print("登录失败,请检查用户名和密码,错误代码")
|
||||
print(e)
|
||||
exit()
|
||||
|
||||
def cat_dir(self, filepath):
|
||||
"""
|
||||
显示ftp指定目录下的文件
|
||||
:param filepath:
|
||||
:return:
|
||||
"""
|
||||
return self.ftp.dir(filepath)
|
||||
|
||||
def make_dir(self, filepath):
|
||||
"""
|
||||
新建ftp下的目录,ftp目录下不存在此文件
|
||||
:param filepath:
|
||||
:return:
|
||||
"""
|
||||
try:
|
||||
filepath.encode("utf-8")
|
||||
mkd = self.ftp.mkd(filepath)
|
||||
return mkd
|
||||
except ftplib.error_perm as e:
|
||||
print("请查看目录下是否存在同名文件,异常为")
|
||||
print(e)
|
||||
|
||||
def remove_dir(self, dirname):
|
||||
"""
|
||||
删除目录
|
||||
:param dirname
|
||||
"""
|
||||
try:
|
||||
self.ftp.rmd(dirname)
|
||||
except ftplib.error_perm as e:
|
||||
print("异常:!!!目录已经被删除了")
|
||||
print(e)
|
||||
|
||||
def download_file(self, localpath, remotepath, filename):
|
||||
"""
|
||||
从FTP指定目录上下载文件,并保存到本地指定路径
|
||||
:param localpath: linux上路径
|
||||
:param remotepath: 本地路径
|
||||
:param filename: 文件名
|
||||
"""
|
||||
bufsize = 8192 # 设置缓冲区大小;
|
||||
ulocalpath = unicode(localpath, "utf-8")
|
||||
try:
|
||||
self.ftp.cwd(remotepath)
|
||||
filepath = ulocalpath + '/'
|
||||
file_handler = open(filepath + filename, "wb").write # 以写模式打开本地文件,---需下载到本地的路径
|
||||
# 传一个回调函数给retrbinary() 它在每接收一个二进制数据时都会被调用
|
||||
self.ftp.retrbinary("RETR " % filename, file_handler, bufsize) # 接收服务器上文件并写入本地文件
|
||||
except ftplib.error_perm as e:
|
||||
print("请检查路径是否正确,是否有权限读取,错误代码")
|
||||
print(e)
|
||||
|
||||
def upload(self, localpath):
|
||||
"""
|
||||
上传至ftp
|
||||
:param localpath: linux上路径
|
||||
"""
|
||||
bufsize = 8192
|
||||
ulocalpath = unicode(localpath, "utf-8")
|
||||
try:
|
||||
file_handler = open(ulocalpath, "rb")
|
||||
filename = os.path.split(ulocalpath)[-1]
|
||||
try:
|
||||
self.ftp.storbinary("stor " % filename, file_handler, bufsize)
|
||||
except ftplib.error_perm as e:
|
||||
print("请检查上传路径是否正确,是否连接FTP,---错误代码")
|
||||
print(e)
|
||||
except e:
|
||||
print("请检查本地路径是否存在····")
|
||||
|
||||
def lst_file(self, path):
|
||||
"""
|
||||
获取指定目录下文件
|
||||
:param path
|
||||
"""
|
||||
s = []
|
||||
try:
|
||||
for name in self.ftp.nlst(path):
|
||||
name1 = unicode(name, "utf-8")
|
||||
s.append(name1)
|
||||
return s
|
||||
except ftplib.all_errors as e:
|
||||
print("请检查目录是否正确,错误代码时-----")
|
||||
print(e)
|
||||
|
||||
def close(self):
|
||||
"""
|
||||
退出ftp
|
||||
"""
|
||||
self.ftp.quit()
|
||||
|
||||
result = []
|
||||
|
||||
# #### 检测传输压缩文件是否完整
|
||||
|
||||
# ##### 1. 融合每天检测
|
||||
|
||||
try:
|
||||
today = dateutils.add_months(datetime.datetime.now(), days=1)
|
||||
date = dateutils.format_date(today, '%Y%m%d')
|
||||
print('当前月:' + date)
|
||||
curr_month_days = calendar.monthrange(today.year, today.month)[1]
|
||||
print('当月天数:' + str(curr_month_days))
|
||||
|
||||
|
||||
# In[79]:
|
||||
|
||||
|
||||
path = '/home/airflow/data/rh'
|
||||
if not os.path.exists(path):
|
||||
os.makedirs(path)
|
||||
|
||||
client = SftpClient()
|
||||
client.sftp_connect(username="ccb", password="ccb!@#", host="192.168.0.45", port=17495)
|
||||
client.get(remote="/home/ccb_rh/output/" + date, local=path)
|
||||
client.disconnect()
|
||||
|
||||
|
||||
# In[80]:
|
||||
|
||||
|
||||
cityids=['440400','210100','469028','330100','441900','440500','370300','320100','530100','140200','370200',
|
||||
'321000','640100','220100','350500','520300','330600','371100','360400','340800','410300','330200',
|
||||
'331000','510100','321100','540100','630100','441200','430100','330500','130900','441800','469005',
|
||||
'620100','130200','610100','330400','340100','440800','440100','131000','321200','350600','440300',
|
||||
'370600','441300','110000','420500','320900','500000','340200','320500','350100','310000','230100',
|
||||
'150100','220200','330300','469029','410100','650100','420100','442000','120000','320200','460200',
|
||||
'130100','450100','450500','430700','370100','520100','450200','320800','320400','320300','320600',
|
||||
'140100','210700','130700','350200','430600','320700','000000','440600','430200','130600','360100',
|
||||
'330700','130300','340500','460100','210200']
|
||||
len(cityids)
|
||||
|
||||
|
||||
# In[81]:
|
||||
|
||||
|
||||
dirs = os.listdir(path + '/' + date)
|
||||
if date + '.ok' not in dirs:
|
||||
result.append(date + '.ok' + '文件不存在')
|
||||
# 每月都要生成
|
||||
if date.endswith('10') == False:
|
||||
name = '000000_' + date + '_FJW'
|
||||
if name + '.tar.gz' not in dirs:
|
||||
result.append(name + '.tar.gz' + '文件不存在')
|
||||
tarutils.untar_file(path + '/' + date + "/" + name + '.tar.gz', path + '/' + date + '/' + name)
|
||||
sub_dirs = os.listdir(path + '/' + date + '/' + name)
|
||||
if 'FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_' + date + '_0001.dat文件不存在')
|
||||
else:
|
||||
cityid = '000000'
|
||||
name = cityid + '_' + date + '_FJW'
|
||||
if name + '.tar.gz' not in dirs:
|
||||
result.append(name + '.tar.gz' + '文件不存在')
|
||||
tarutils.untar_file(path + '/' + date + "/" + name + '.tar.gz', path + '/' + date + '/' + name)
|
||||
# 全国
|
||||
sub_dirs = os.listdir(path + '/' + date + '/' + name)
|
||||
if 'FJW_000000_' + cityid + '_EVALUATE_RATE_FJW_ALL_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('FJW_000000_' + cityid + '_EVALUATE_RATE_FJW_ALL_' + date + '_0001.dat文件不存在')
|
||||
|
||||
## 固定13个文件
|
||||
if 'FJW_000000_' + cityid + '_ADDRESS_ALIAS_FJW_ALL_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('FJW_000000_' + cityid + '_ADDRESS_ALIAS_FJW_ALL_' + date + '_0001.dat文件不存在')
|
||||
if 'FJW_000000_' + cityid + '_BIGDATA_BUILDING_FJW_ALL_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('FJW_000000_' + cityid + '_BIGDATA_BUILDING_FJW_ALL_' + date + '_0001.dat文件不存在')
|
||||
if 'FJW_000000_' + cityid + '_BIGDATA_BUILDPROJ_FJW_ALL_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('FJW_000000_' + cityid + '_BIGDATA_BUILDPROJ_FJW_ALL_' + date + '_0001.dat文件不存在')
|
||||
if 'FJW_000000_' + cityid + '_BIGDATA_COMMBUILDSTRU_FJW_ALL_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('FJW_000000_' + cityid + '_BIGDATA_COMMBUILDSTRU_FJW_ALL_' + date + '_0001.dat文件不存在')
|
||||
if 'FJW_000000_' + cityid + '_BIGDATA_COMMBUILDYEAR_FJW_ALL_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('FJW_000000_' + cityid + '_BIGDATA_COMMBUILDYEAR_FJW_ALL_' + date + '_0001.dat文件不存在')
|
||||
|
||||
if 'FJW_000000_' + cityid + '_BIGDATA_COMMFEE_FJW_ALL_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('FJW_000000_' + cityid + '_BIGDATA_COMMFEE_FJW_ALL_' + date + '_0001.dat文件不存在')
|
||||
if 'FJW_000000_' + cityid + '_BIGDATA_COMMHOUSETYPE_FJW_ALL_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('FJW_000000_' + cityid + '_BIGDATA_COMMHOUSETYPE_FJW_ALL_' + date + '_0001.dat文件不存在')
|
||||
if 'FJW_000000_' + cityid + '_BIGDATA_COMMHOUSEUSE_FJW_ALL_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('FJW_000000_' + cityid + '_BIGDATA_COMMHOUSEUSE_FJW_ALL_' + date + '_0001.dat文件不存在')
|
||||
if 'FJW_000000_' + cityid + '_BIGDATA_COMMLAYOUTPIC_FJW_ALL_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('FJW_000000_' + cityid + '_BIGDATA_COMMLAYOUTPIC_FJW_ALL_' + date + '_0001.dat文件不存在')
|
||||
if 'FJW_000000_' + cityid + '_BIGDATA_COMMUNITY_FJW_ALL_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('FJW_000000_' + cityid + '_BIGDATA_COMMUNITY_FJW_ALL_' + date + '_0001.dat文件不存在')
|
||||
|
||||
if 'FJW_000000_' + cityid + '_BIGDATA_DISTRICT_FJW_ALL_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('FJW_000000_' + cityid + '_BIGDATA_DISTRICT_FJW_ALL_' + date + '_0001.dat文件不存在')
|
||||
if 'FJW_000000_' + cityid + '_BIGDATA_POSITION_FJW_ALL_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('FJW_000000_' + cityid + '_BIGDATA_POSITION_FJW_ALL_' + date + '_0001.dat文件不存在')
|
||||
if 'FJW_000000_' + cityid + '_BIGDATA_DEAL_FJW_ADD_' + date + '_0001.dat' not in sub_dirs:
|
||||
result.append('FJW_000000_' + cityid + '_BIGDATA_DEAL_FJW_ADD_' + date + '_0001.dat文件不存在')
|
||||
if len(result) == 0:
|
||||
result.append('融合检测完毕未发现异常文件')
|
||||
shutil.rmtree(path + "/" + date)
|
||||
|
||||
except Exception as ex:
|
||||
result.append(str(ex))
|
||||
|
||||
|
||||
# In[82]:
|
||||
|
||||
|
||||
print(result)
|
||||
|
||||
|
||||
|
||||
# In[76]:
|
||||
|
||||
|
||||
infos = "【" + date + "自动检测】"
|
||||
for item in result:
|
||||
infos += item + "\n"
|
||||
data = {
|
||||
"msgtype": "text",
|
||||
"text": {
|
||||
"content": infos
|
||||
}
|
||||
}
|
||||
# 发送消息至群
|
||||
requests.post(url='https://oapi.dingtalk.com/robot/send?access_token=1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736',
|
||||
json=data)
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
from airflow.operators.bash import BashOperator
|
||||
import pendulum
|
||||
from airflow import DAG
|
||||
|
||||
with DAG(
|
||||
dag_id="merge_dag",
|
||||
# 每日下午4点
|
||||
schedule_interval="0 16 * * *",
|
||||
start_date=pendulum.datetime(2022, 8, 23, tz="Asia/Shanghai"),
|
||||
catchup=True,
|
||||
description="每日融合文件检测"
|
||||
) as dag:
|
||||
t1 = BashOperator(
|
||||
task_id='merge',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/jh/merge/merge.py',
|
||||
)
|
||||
t1
|
||||
Reference in New Issue
Block a user