#!/usr/bin/env python # coding: utf-8 from goojia_utils import dateutils, ftputils, tarutils import datetime import shutil import os import requests import logging # #### 检测传输压缩文件是否完整 # ##### 2.估值每月27日检测 # In[60]: log = logging.getLogger(__name__) date = dateutils.format_date(datetime.datetime.now(), '%Y%m') + '28' path = '/home/airflow/data/gz' local_aim_dir = path + '/' + date if not os.path.exists(path): os.makedirs(path) if not os.path.exists(local_aim_dir): os.makedirs(local_aim_dir) client = ftputils.SftpClient() client.sftp_connect(username="ccb", password="ccb!@#", host="192.168.0.45", port=17495) client.get(remote="/home/ccb_evaluation/output/" + date + '/WS_GJ_000000_' + date + '_0001.tar', local=local_aim_dir) client.get(remote="/home/ccb_evaluation/output/" + date + '/' + date + '.ok', local=path + '/' + date) client.get(remote="/home/ccb_evaluation/output/" + date + '/WS_GJ_000000_' + date + '_0002.tar', local=local_aim_dir) client.disconnect() # In[57]: ## 解压 result = [] try: dirs = os.listdir(local_aim_dir) if date + '.ok' not in dirs: result.append(date + '.ok' + '文件不存在') tar_file = 'WS_GJ_000000_' + date + '_0001.tar' has_two_tar = False if 'WS_GJ_000000_' + date + '_0002.tar' in dirs: has_two_tar = True shutil.rmtree(local_aim_dir + '/' + date + '_2') tarutils.untar_file(local_aim_dir + '/WS_GJ_000000_' + date + '_0002.tar', local_aim_dir + '/') if tar_file not in dirs: result.append(tar_file + '文件不存在') else: if has_two_tar is True: shutil.rmtree(local_aim_dir + '/' + date + '_1') tarutils.untar_file(local_aim_dir + "/" + tar_file, local_aim_dir + '/') if has_two_tar is False: sub_dirs = os.listdir(local_aim_dir) else: sub_dirs = os.listdir(local_aim_dir + '/' + date + '_1') sub_dirs.extend(os.listdir(local_aim_dir + '/' + date + '_2')) if 'MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_' + date + '_0001.dat' not in sub_dirs: result.append('MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_' + date + '_0001.dat文件不存在') if 'MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_' + date + '_0001.dat' not in sub_dirs: result.append('MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_' + date + '_0001.dat文件不存在') if 'MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_' + date + '_0001.dat' not in sub_dirs: result.append('MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_' + date + '_0001.dat文件不存在') if 'MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_' + date + '_0001.dat' not in sub_dirs: result.append('MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_' + date + '_0001.dat文件不存在') if 'MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_' + date + '_0001.dat' not in sub_dirs: result.append('MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_' + date + '_0001.dat文件不存在') if 'MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_' + date + '_0001.dat' not in sub_dirs: result.append('MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_' + date + '_0001.dat文件不存在') if 'MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_' + date + '_0001.dat' not in sub_dirs: result.append('MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_' + date + '_0001.dat文件不存在') if 'MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_' + date + '_0001.dat' not in sub_dirs: result.append('MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_' + date + '_0001.dat文件不存在') if 'MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_' + date + '_0001.dat' not in sub_dirs: result.append('MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_' + date + '_0001.dat文件不存在') if 'MODEL_000000_000000_BIGDATA_POI_GJ_ALL_' + date + '_0001.dat' not in sub_dirs: result.append('MODEL_000000_000000_BIGDATA_POI_GJ_ALL_' + date + '_0001.dat文件不存在') if 'MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_' + date + '_0001.dat' not in sub_dirs: result.append('MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_' + date + '_0001.dat文件不存在') if 'MODEL_000000_000000_CITY_PRICE_GJ_ADD_' + date + '_0001.dat' not in sub_dirs: result.append('MODEL_000000_000000_CITY_PRICE_GJ_ADD_' + date + '_0001.dat文件不存在') if 'MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_' + date + '_0001.dat' not in sub_dirs: result.append('MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_' + date + '_0001.dat文件不存在') if 'MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_' + date + '_0001.dat' not in sub_dirs: result.append('MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_' + date + '_0001.dat文件不存在') if 'MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_' + date + '_0001.dat' not in sub_dirs: result.append('MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_' + date + '_0001.dat文件不存在') if 'MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_' + date + '_0001.dat' not in sub_dirs: result.append('MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_' + date + '_0001.dat文件不存在') if 'MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_' + date + '_0001.dat' not in sub_dirs: result.append('MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_' + date + '_0001.dat文件不存在') if 'MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_' + date + '_0001.dat' not in sub_dirs: result.append('MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_' + date + '_0001.dat文件不存在') if 'MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_' + date + '_0001.dat' not in sub_dirs: result.append('MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_' + date + '_0001.dat文件不存在') if 'MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_' + date + '_0001.dat' not in sub_dirs: result.append('MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_' + date + '_0001.dat文件不存在') if 'MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_' + date + '_0001.dat' not in sub_dirs: result.append('MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_' + date + '_0001.dat文件不存在') if len(result) == 0: result.append('估值检测完毕未发现异常文件') shutil.rmtree(local_aim_dir) else: result.append(" @高娟 ") except Exception as ex: result.append(" @高娟 ") result.append(str(ex)) infos = "【" + date + "自动检测】" for item in result: infos += item + "\n" data = { "msgtype": "text", "text": { "content": infos } } log.info(data) # 发送消息至群 requests.post(url='https://oapi.dingtalk.com/robot/send?access_token=1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736', json=data)