Files
airflow/dags/jh/merge/merge.py
T
2024-05-09 16:06:51 +08:00

163 lines
9.3 KiB
Python

from goojia_utils import dateutils, ftputils, tarutils
import datetime
import requests
import shutil
import os
import sys
import logging
log = logging.getLogger(__name__)
result = []
# ##### 1. 融合每天检测
try:
date = sys.argv[1] if len(sys.argv) > 1 else None
if date is None:
today = dateutils.add_months(datetime.datetime.now(), days=1)
date = dateutils.format_date(today, '%Y%m%d')
print('当前月:' + date)
# curr_month_days = calendar.monthrange(today.year, today.month)[1]
# print('当月天数:' + str(curr_month_days))
# In[79]:
path = '/home/airflow/data/rh'
if not os.path.exists(path):
os.makedirs(path)
if not os.path.exists(path + "/" + date):
os.makedirs(path + "/" + date)
client = ftputils.SftpClient()
client.sftp_connect(username="ccb", password="ccb!@#", host="192.168.0.45", port=17495)
client.get(remote="/home/ccb_rh/output/" + date + '/000000_' + date + '_FJW.tar.gz', local=path + '/' + date)
client.get(remote="/home/ccb_rh/output/" + date + '/' + date + '.ok', local=path + '/' + date)
client.disconnect()
# In[80]:
cityids=['440400','210100','469028','330100','441900','440500','370300','320100','530100','140200','370200',
'321000','640100','220100','350500','520300','330600','371100','360400','340800','410300','330200',
'331000','510100','321100','540100','630100','441200','430100','330500','130900','441800','469005',
'620100','130200','610100','330400','340100','440800','440100','131000','321200','350600','440300',
'370600','441300','110000','420500','320900','500000','340200','320500','350100','310000','230100',
'150100','220200','330300','469029','410100','650100','420100','442000','120000','320200','460200',
'130100','450100','450500','430700','370100','520100','450200','320800','320400','320300','320600',
'140100','210700','130700','350200','430600','320700','000000','440600','430200','130600','360100',
'330700','130300','340500','460100','210200']
len(cityids)
# In[81]:
dirs = os.listdir(path + '/' + date)
if date + '.ok' not in dirs:
result.append(date + '.ok' + '文件不存在')
# 每月都要生成
if date.endswith('10') == False:
name = '000000_' + date + '_FJW'
if name + '.tar.gz' not in dirs:
result.append(name + '.tar.gz' + '文件不存在')
tarutils.untar_file(path + '/' + date + "/" + name + '.tar.gz', path + '/' + date + '/' + name)
sub_dirs = os.listdir(path + '/' + date + '/' + name)
if 'FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_' + date + '_0001.dat文件不存在')
else:
cityid = '000000'
name = cityid + '_' + date + '_FJW'
if name + '.tar.gz' not in dirs:
result.append(name + '.tar.gz' + '文件不存在')
tarutils.untar_file(path + '/' + date + "/" + name + '.tar.gz', path + '/' + date + '/' + name)
# 全国
sub_dirs = os.listdir(path + '/' + date + '/' + name)
if 'FJW_000000_' + cityid + '_EVALUATE_RATE_FJW_ALL_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_' + cityid + '_EVALUATE_RATE_FJW_ALL_' + date + '_0001.dat文件不存在')
## 固定13个文件
if 'FJW_000000_' + cityid + '_ADDRESS_ALIAS_FJW_ALL_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_' + cityid + '_ADDRESS_ALIAS_FJW_ALL_' + date + '_0001.dat文件不存在')
if 'FJW_000000_' + cityid + '_BIGDATA_BUILDING_FJW_ALL_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_' + cityid + '_BIGDATA_BUILDING_FJW_ALL_' + date + '_0001.dat文件不存在')
if 'FJW_000000_' + cityid + '_BIGDATA_BUILDPROJ_FJW_ALL_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_' + cityid + '_BIGDATA_BUILDPROJ_FJW_ALL_' + date + '_0001.dat文件不存在')
if 'FJW_000000_' + cityid + '_BIGDATA_COMMBUILDSTRU_FJW_ALL_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_' + cityid + '_BIGDATA_COMMBUILDSTRU_FJW_ALL_' + date + '_0001.dat文件不存在')
if 'FJW_000000_' + cityid + '_BIGDATA_COMMBUILDYEAR_FJW_ALL_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_' + cityid + '_BIGDATA_COMMBUILDYEAR_FJW_ALL_' + date + '_0001.dat文件不存在')
if 'FJW_000000_' + cityid + '_BIGDATA_COMMFEE_FJW_ALL_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_' + cityid + '_BIGDATA_COMMFEE_FJW_ALL_' + date + '_0001.dat文件不存在')
if 'FJW_000000_' + cityid + '_BIGDATA_COMMHOUSETYPE_FJW_ALL_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_' + cityid + '_BIGDATA_COMMHOUSETYPE_FJW_ALL_' + date + '_0001.dat文件不存在')
if 'FJW_000000_' + cityid + '_BIGDATA_COMMHOUSEUSE_FJW_ALL_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_' + cityid + '_BIGDATA_COMMHOUSEUSE_FJW_ALL_' + date + '_0001.dat文件不存在')
if 'FJW_000000_' + cityid + '_BIGDATA_COMMLAYOUTPIC_FJW_ALL_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_' + cityid + '_BIGDATA_COMMLAYOUTPIC_FJW_ALL_' + date + '_0001.dat文件不存在')
if 'FJW_000000_' + cityid + '_BIGDATA_COMMUNITY_FJW_ALL_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_' + cityid + '_BIGDATA_COMMUNITY_FJW_ALL_' + date + '_0001.dat文件不存在')
if 'FJW_000000_' + cityid + '_BIGDATA_DISTRICT_FJW_ALL_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_' + cityid + '_BIGDATA_DISTRICT_FJW_ALL_' + date + '_0001.dat文件不存在')
if 'FJW_000000_' + cityid + '_BIGDATA_POSITION_FJW_ALL_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_' + cityid + '_BIGDATA_POSITION_FJW_ALL_' + date + '_0001.dat文件不存在')
if 'FJW_000000_' + cityid + '_BIGDATA_DEAL_FJW_ADD_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_' + cityid + '_BIGDATA_DEAL_FJW_ADD_' + date + '_0001.dat文件不存在')
if 'FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_' + date + '_0001.dat文件不存在')
if 'FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_' + date + '_0001.dat文件不存在')
if 'FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_' + date + '_0001.dat文件不存在')
if 'FJW_000000_000000_EVALUATE_RATE_CITY_FJW_HIS_ADD_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_000000_EVALUATE_RATE_CITY_FJW_HIS_ADD_' + date + '_0001.dat文件不存在')
if 'FJW_000000_000000_EVALUATE_RATE_COMM_FJW_HIS_ADD_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_000000_EVALUATE_RATE_COMM_FJW_HIS_ADD_' + date + '_0001.dat文件不存在')
# if 'FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_' + date + '_0001.dat' not in sub_dirs:
# result.append('FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_' + date + '_0001.dat文件不存在')
if date.endswith('28'):
if 'FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_' + date + '_0001.dat文件不存在')
if 'FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_' + date + '_0001.dat文件不存在')
if 'FJW_000000_000000_COMM_BASE_PRICE_GJ_ADD_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_000000_COMM_BASE_PRICE_GJ_ADD_' + date + '_0001.dat文件不存在')
if 'FJW_000000_000000_COMM_DESCRIBE_GJ_ADD_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_000000_COMM_DESCRIBE_GJ_ADD_' + date + '_0001.dat文件不存在')
if 'FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_' + date + '_0001.dat文件不存在')
if 'FJW_000000_000000_GUIDE_STATISTICS_ADD_' + date + '_0001.dat' not in sub_dirs:
result.append('FJW_000000_000000_GUIDE_STATISTICS_ADD_' + date + '_0001.dat文件不存在')
if len(result) == 0:
result.append('融合检测完毕未发现异常文件')
shutil.rmtree(path + "/" + date)
else:
result.append(" @高娟 ")
except Exception as ex:
result.append(" @高娟 ")
result.append(str(ex))
infos = "【" + date + "自动检测】"
for item in result:
infos += item + "\n"
data = {
"msgtype": "text",
"text": {
"content": infos
}
}
log.info(data)
# 发送消息至群
requests.post(url='https://oapi.dingtalk.com/robot/send?access_token=1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736',
json=data)