198 lines
6.8 KiB
Python
198 lines
6.8 KiB
Python
# -*- coding: utf-8 -*-
|
||
import os
|
||
import datetime
|
||
import calendar
|
||
import json
|
||
import pandas as pd
|
||
from goojia_utils import LOG_MONGO_CLIENT
|
||
from airflow import DAG
|
||
from airflow.operators.python import PythonOperator
|
||
from airflow.utils.email import send_email
|
||
import pendulum
|
||
|
||
|
||
def get_last_month_range(ref_date: datetime.date = None):
|
||
"""
|
||
获取参考日期的上个月第一天 00:00:00 与最后一天 23:59:59.999999
|
||
以北京时间为准
|
||
"""
|
||
if ref_date is None:
|
||
ref_date = datetime.date.today()
|
||
|
||
first_day_current_month = ref_date.replace(day=1)
|
||
last_day_last_month = first_day_current_month - datetime.timedelta(days=1)
|
||
first_day_last_month = last_day_last_month.replace(day=1)
|
||
|
||
start_time = datetime.datetime.combine(first_day_last_month, datetime.time.min)
|
||
end_time = datetime.datetime.combine(last_day_last_month, datetime.time.max)
|
||
|
||
return start_time, end_time, first_day_last_month.year, first_day_last_month.month
|
||
|
||
|
||
def parse_dict(val):
|
||
"""将字符串类型的 JSON 或字典解析为 dict"""
|
||
if isinstance(val, dict):
|
||
return val
|
||
if isinstance(val, str) and val.strip():
|
||
try:
|
||
return json.loads(val)
|
||
except Exception:
|
||
return {}
|
||
return {}
|
||
|
||
|
||
# Excel 中文表头定义(已剔除:涉农类型、API访问授权凭证、居委、居委编码、城乡代码、城乡代码对应中文)
|
||
EXCEL_COLUMNS = [
|
||
'请求时间',
|
||
# 输入参数
|
||
'城市',
|
||
'地址',
|
||
# 响应参数
|
||
'省份',
|
||
'省份编码',
|
||
'城市名称',
|
||
'城市编码',
|
||
'区域',
|
||
'区域编码',
|
||
'乡镇街道',
|
||
'乡镇街道编码',
|
||
'是否涉农',
|
||
'所属区域',
|
||
'地址验证',
|
||
'是否为县级市',
|
||
'是否县城城区',
|
||
'是否为省国家级开发区',
|
||
'省国家级开发区名称',
|
||
'开发区等级'
|
||
]
|
||
|
||
|
||
def export_and_send_njcb_statement(**context):
|
||
"""
|
||
查询上月对公涉农API日志,生成全中文表头Excel对账单并通过邮件发送给南京银行顾老师
|
||
"""
|
||
now = datetime.datetime.now()
|
||
start_time, end_time, last_year, last_month = get_last_month_range(now.date())
|
||
|
||
print(f"统计时间范围: {start_time} 至 {end_time} (归属年月: {last_year}年{last_month:02d}月)")
|
||
|
||
open_uri_log_conn = LOG_MONGO_CLIENT.get_collection('business', 'open_uri_log')
|
||
|
||
# 查询条件:
|
||
# user_id = 1007801, request_uri = /addrSvc/njcbVillageIdentify, c_date在时间范围内, 且只输出code是200的响应
|
||
query = {
|
||
'user_id': '1007801',
|
||
'request_uri': '/addrSvc/njcbVillageIdentify',
|
||
'c_date': {
|
||
'$gte': start_time,
|
||
'$lte': end_time
|
||
},
|
||
'$or': [
|
||
{'code': 200},
|
||
{'code': '200'}
|
||
]
|
||
}
|
||
|
||
print(f"开始查询 MongoDB business.open_uri_log,条件: {query}")
|
||
cursor = open_uri_log_conn.find(query).batch_size(2000)
|
||
|
||
rows = []
|
||
scanned_count = 0
|
||
for doc in cursor:
|
||
scanned_count += 1
|
||
param_map = parse_dict(doc.get('parameter_map'))
|
||
res_map = parse_dict(doc.get('res_map'))
|
||
|
||
# 过滤只保留"对公涉农"的请求
|
||
# 涉农类型: 1 对私,2 对公 (默认为对私)
|
||
req_type = str(param_map.get('type', '1')).strip()
|
||
if req_type != '2':
|
||
continue
|
||
|
||
c_date = doc.get('c_date')
|
||
if isinstance(c_date, datetime.datetime):
|
||
c_date_str = c_date.strftime('%Y-%m-%d %H:%M:%S')
|
||
else:
|
||
c_date_str = str(c_date) if c_date is not None else ''
|
||
|
||
row = {
|
||
'请求时间': c_date_str,
|
||
# 输入参数
|
||
'城市': param_map.get('city', ''),
|
||
'地址': param_map.get('address', ''),
|
||
# 响应参数
|
||
'省份': res_map.get('province', ''),
|
||
'省份编码': res_map.get('provinceCode', ''),
|
||
'城市名称': res_map.get('city', ''),
|
||
'城市编码': res_map.get('cityCode', ''),
|
||
'区域': res_map.get('district', ''),
|
||
'区域编码': res_map.get('districtCode', ''),
|
||
'乡镇街道': res_map.get('town', ''),
|
||
'乡镇街道编码': res_map.get('townCode', ''),
|
||
'是否涉农': res_map.get('verifyFlag', ''),
|
||
'所属区域': res_map.get('addrArea', ''),
|
||
'地址验证': res_map.get('addrVerify', ''),
|
||
'是否为县级市': res_map.get('countyLevelCity', ''),
|
||
'是否县城城区': res_map.get('countyGov', ''),
|
||
'是否为省国家级开发区': res_map.get('devDistrict', ''),
|
||
'省国家级开发区名称': res_map.get('devDistrictName', ''),
|
||
'开发区等级': res_map.get('devLevel', '')
|
||
}
|
||
rows.append(row)
|
||
|
||
total_count = len(rows)
|
||
print(f"扫描命中记录: {scanned_count} 条,其中符合对公涉农(type=2)记录: {total_count} 条")
|
||
|
||
# 生成 Excel 文件
|
||
tmp_dir = os.path.join(os.path.expanduser('~'), 'tmp', 'njcb_statement')
|
||
os.makedirs(tmp_dir, exist_ok=True)
|
||
filename = f"南京银行涉农API对账单_{last_year}{last_month:02d}.xlsx"
|
||
excel_path = os.path.join(tmp_dir, filename)
|
||
|
||
if rows:
|
||
df = pd.DataFrame(rows, columns=EXCEL_COLUMNS)
|
||
else:
|
||
df = pd.DataFrame(columns=EXCEL_COLUMNS)
|
||
|
||
df.to_excel(excel_path, index=False, engine='openpyxl')
|
||
print(f"Excel文件生成成功: {excel_path}")
|
||
|
||
# 发送邮件
|
||
receiver = 'wangwenyan@fangjia.com'
|
||
subject = f"南京银行{last_year}年{last_month:02d}月涉农API对账单"
|
||
html_content = f"""
|
||
<h3>南京银行 顾老师:</h3>
|
||
<p>您好!</p>
|
||
<p>附件为南京银行 <b>{last_year}年{last_month:02d}月</b> 涉农识别API(/addrSvc/njcbVillageIdentify)对公涉农对账单,请查收。</p>
|
||
<ul>
|
||
<li><b>对账周期:</b> {start_time.strftime('%Y-%m-%d')} 至 {end_time.strftime('%Y-%m-%d')}</li>
|
||
<li><b>调用有效成功条数(Code=200 且 对公涉农):</b> {total_count} 条</li>
|
||
</ul>
|
||
<p>如有任何疑问,请随时与我们联系。</p>
|
||
"""
|
||
|
||
print(f"正在发送邮件至: {receiver}, 附件: {excel_path}")
|
||
send_email(
|
||
to=receiver,
|
||
subject=subject,
|
||
html_content=html_content,
|
||
files=[excel_path]
|
||
)
|
||
print("邮件发送完成。")
|
||
|
||
|
||
# Airflow DAG 定义:每月2号上午9:00执行(北京时间)
|
||
with DAG(
|
||
dag_id="njcb_village_identify_statement_dag",
|
||
schedule="0 9 2 * *",
|
||
start_date=pendulum.datetime(2024, 1, 1, tz="Asia/Shanghai"),
|
||
catchup=False,
|
||
description="每月2号给南京银行顾老师发送上个月涉农API对账单",
|
||
tags=["api", "njcb", "statement"]
|
||
) as dag:
|
||
|
||
send_statement_task = PythonOperator(
|
||
task_id="send_njcb_village_statement",
|
||
python_callable=export_and_send_njcb_statement
|
||
)
|