Files
airflow/dags/api/dag_njcb_village_statement.py
T

198 lines
6.8 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
# -*- coding: utf-8 -*-
import os
import datetime
import calendar
import json
import pandas as pd
from goojia_utils import LOG_MONGO_CLIENT
from airflow import DAG
from airflow.operators.python import PythonOperator
from airflow.utils.email import send_email
import pendulum
def get_last_month_range(ref_date: datetime.date = None):
"""
获取参考日期的上个月第一天 00:00:00 与最后一天 23:59:59.999999
以北京时间为准
"""
if ref_date is None:
ref_date = datetime.date.today()
first_day_current_month = ref_date.replace(day=1)
last_day_last_month = first_day_current_month - datetime.timedelta(days=1)
first_day_last_month = last_day_last_month.replace(day=1)
start_time = datetime.datetime.combine(first_day_last_month, datetime.time.min)
end_time = datetime.datetime.combine(last_day_last_month, datetime.time.max)
return start_time, end_time, first_day_last_month.year, first_day_last_month.month
def parse_dict(val):
"""将字符串类型的 JSON 或字典解析为 dict"""
if isinstance(val, dict):
return val
if isinstance(val, str) and val.strip():
try:
return json.loads(val)
except Exception:
return {}
return {}
# Excel 中文表头定义(已剔除:涉农类型、API访问授权凭证、居委、居委编码、城乡代码、城乡代码对应中文)
EXCEL_COLUMNS = [
'请求时间',
# 输入参数
'城市',
'地址',
# 响应参数
'省份',
'省份编码',
'城市名称',
'城市编码',
'区域',
'区域编码',
'乡镇街道',
'乡镇街道编码',
'是否涉农',
'所属区域',
'地址验证',
'是否为县级市',
'是否县城城区',
'是否为省国家级开发区',
'省国家级开发区名称',
'开发区等级'
]
def export_and_send_njcb_statement(**context):
"""
查询上月对公涉农API日志,生成全中文表头Excel对账单并通过邮件发送给南京银行顾老师
"""
now = datetime.datetime.now()
start_time, end_time, last_year, last_month = get_last_month_range(now.date())
print(f"统计时间范围: {start_time} 至 {end_time} (归属年月: {last_year}年{last_month:02d}月)")
open_uri_log_conn = LOG_MONGO_CLIENT.get_collection('business', 'open_uri_log')
# 查询条件:
# user_id = 1007801, request_uri = /addrSvc/njcbVillageIdentify, c_date在时间范围内, 且只输出code是200的响应
query = {
'user_id': '1007801',
'request_uri': '/addrSvc/njcbVillageIdentify',
'c_date': {
'$gte': start_time,
'$lte': end_time
},
'$or': [
{'code': 200},
{'code': '200'}
]
}
print(f"开始查询 MongoDB business.open_uri_log,条件: {query}")
cursor = open_uri_log_conn.find(query).batch_size(2000)
rows = []
scanned_count = 0
for doc in cursor:
scanned_count += 1
param_map = parse_dict(doc.get('parameter_map'))
res_map = parse_dict(doc.get('res_map'))
# 过滤只保留"对公涉农"的请求
# 涉农类型: 1 对私,2 对公 (默认为对私)
req_type = str(param_map.get('type', '1')).strip()
if req_type != '2':
continue
c_date = doc.get('c_date')
if isinstance(c_date, datetime.datetime):
c_date_str = c_date.strftime('%Y-%m-%d %H:%M:%S')
else:
c_date_str = str(c_date) if c_date is not None else ''
row = {
'请求时间': c_date_str,
# 输入参数
'城市': param_map.get('city', ''),
'地址': param_map.get('address', ''),
# 响应参数
'省份': res_map.get('province', ''),
'省份编码': res_map.get('provinceCode', ''),
'城市名称': res_map.get('city', ''),
'城市编码': res_map.get('cityCode', ''),
'区域': res_map.get('district', ''),
'区域编码': res_map.get('districtCode', ''),
'乡镇街道': res_map.get('town', ''),
'乡镇街道编码': res_map.get('townCode', ''),
'是否涉农': res_map.get('verifyFlag', ''),
'所属区域': res_map.get('addrArea', ''),
'地址验证': res_map.get('addrVerify', ''),
'是否为县级市': res_map.get('countyLevelCity', ''),
'是否县城城区': res_map.get('countyGov', ''),
'是否为省国家级开发区': res_map.get('devDistrict', ''),
'省国家级开发区名称': res_map.get('devDistrictName', ''),
'开发区等级': res_map.get('devLevel', '')
}
rows.append(row)
total_count = len(rows)
print(f"扫描命中记录: {scanned_count} 条,其中符合对公涉农(type=2)记录: {total_count} 条")
# 生成 Excel 文件
tmp_dir = os.path.join(os.path.expanduser('~'), 'tmp', 'njcb_statement')
os.makedirs(tmp_dir, exist_ok=True)
filename = f"南京银行涉农API对账单_{last_year}{last_month:02d}.xlsx"
excel_path = os.path.join(tmp_dir, filename)
if rows:
df = pd.DataFrame(rows, columns=EXCEL_COLUMNS)
else:
df = pd.DataFrame(columns=EXCEL_COLUMNS)
df.to_excel(excel_path, index=False, engine='openpyxl')
print(f"Excel文件生成成功: {excel_path}")
# 发送邮件
receiver = 'wangwenyan@fangjia.com'
subject = f"南京银行{last_year}年{last_month:02d}月涉农API对账单"
html_content = f"""
<h3>南京银行 顾老师:</h3>
<p>您好!</p>
<p>附件为南京银行 <b>{last_year}年{last_month:02d}月</b> 涉农识别API(/addrSvc/njcbVillageIdentify)对公涉农对账单,请查收。</p>
<ul>
<li><b>对账周期:</b> {start_time.strftime('%Y-%m-%d')} 至 {end_time.strftime('%Y-%m-%d')}</li>
<li><b>调用有效成功条数(Code=200 且 对公涉农):</b> {total_count} 条</li>
</ul>
<p>如有任何疑问,请随时与我们联系。</p>
"""
print(f"正在发送邮件至: {receiver}, 附件: {excel_path}")
send_email(
to=receiver,
subject=subject,
html_content=html_content,
files=[excel_path]
)
print("邮件发送完成。")
# Airflow DAG 定义:每月2号上午9:00执行(北京时间)
with DAG(
dag_id="njcb_village_identify_statement_dag",
schedule="0 9 2 * *",
start_date=pendulum.datetime(2024, 1, 1, tz="Asia/Shanghai"),
catchup=False,
description="每月2号给南京银行顾老师发送上个月涉农API对账单",
tags=["api", "njcb", "statement"]
) as dag:
send_statement_task = PythonOperator(
task_id="send_njcb_village_statement",
python_callable=export_and_send_njcb_statement
)