From df9dd814878ca09dd26750e83e8998f6f0add00c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=96=87=E8=89=B3?= Date: Wed, 16 Sep 2026 08:50:41 +0800 Subject: [PATCH] =?UTF-8?q?refactor(api):=20=E6=8B=86=E5=88=86=E5=8D=97?= =?UTF-8?q?=E4=BA=AC=E9=93=B6=E8=A1=8C=E6=B6=89=E5=86=9CAPI=E5=AF=B9?= =?UTF-8?q?=E8=B4=A6=E5=8D=95DAG=E4=B8=8E=E6=89=A7=E8=A1=8C=E8=84=9A?= =?UTF-8?q?=E6=9C=AC=E4=B8=BABashOperator=E6=A8=A1=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/api/api/njcb_village_statement.py | 181 +++++++++++++++++++++++ dags/api/dag_njcb_village_statement.py | 189 ++----------------------- 2 files changed, 190 insertions(+), 180 deletions(-) create mode 100644 dags/api/api/njcb_village_statement.py diff --git a/dags/api/api/njcb_village_statement.py b/dags/api/api/njcb_village_statement.py new file mode 100644 index 0000000..15fef71 --- /dev/null +++ b/dags/api/api/njcb_village_statement.py @@ -0,0 +1,181 @@ +import os +import datetime +import calendar +import json +import pandas as pd +from goojia_utils import LOG_MONGO_CLIENT +from airflow.utils.email import send_email + + +def get_last_month_range(ref_date: datetime.date = None): + """ + 获取参考日期的上个月第一天 00:00:00 与最后一天 23:59:59.999999 + 以北京时间为准 + """ + if ref_date is None: + ref_date = datetime.date.today() + + first_day_current_month = ref_date.replace(day=1) + last_day_last_month = first_day_current_month - datetime.timedelta(days=1) + first_day_last_month = last_day_last_month.replace(day=1) + + start_time = datetime.datetime.combine(first_day_last_month, datetime.time.min) + end_time = datetime.datetime.combine(last_day_last_month, datetime.time.max) + + return start_time, end_time, first_day_last_month.year, first_day_last_month.month + + +def parse_dict(val): + """将字符串类型的 JSON 或字典解析为 dict""" + if isinstance(val, dict): + return val + if isinstance(val, str) and val.strip(): + try: + return json.loads(val) + except Exception: + return {} + return {} + + +# Excel 中文表头定义(已剔除:涉农类型、API访问授权凭证、居委、居委编码、城乡代码、城乡代码对应中文) +EXCEL_COLUMNS = [ + '请求时间', + # 输入参数 + '城市', + '地址', + # 响应参数 + '省份', + '省份编码', + '城市名称', + '城市编码', + '区域', + '区域编码', + '乡镇街道', + '乡镇街道编码', + '是否涉农', + '所属区域', + '地址验证', + '是否为县级市', + '是否县城城区', + '是否为省国家级开发区', + '省国家级开发区名称', + '开发区等级' +] + + +def export_and_send_njcb_statement(): + """ + 查询上月对公涉农API日志,生成全中文表头Excel对账单并通过邮件发送给南京银行顾老师 + """ + now = datetime.datetime.now() + start_time, end_time, last_year, last_month = get_last_month_range(now.date()) + + print(f"统计时间范围: {start_time} 至 {end_time} (归属年月: {last_year}年{last_month:02d}月)") + + open_uri_log_conn = LOG_MONGO_CLIENT.get_collection('business', 'open_uri_log') + + # 查询条件: + # user_id = 1007801, request_uri = /addrSvc/njcbVillageIdentify, c_date在时间范围内, 且只输出code是200的响应 + query = { + 'user_id': '1007801', + 'request_uri': '/addrSvc/njcbVillageIdentify', + 'c_date': { + '$gte': start_time, + '$lte': end_time + }, + '$or': [ + {'code': 200}, + {'code': '200'} + ] + } + + print(f"开始查询 MongoDB business.open_uri_log,条件: {query}") + cursor = open_uri_log_conn.find(query).batch_size(2000) + + rows = [] + scanned_count = 0 + for doc in cursor: + scanned_count += 1 + param_map = parse_dict(doc.get('parameter_map')) + res_map = parse_dict(doc.get('res_map')) + + # 过滤只保留"对公涉农"的请求 + # 涉农类型: 1 对私,2 对公 (默认为对私) + req_type = str(param_map.get('type', '1')).strip() + if req_type != '2': + continue + + c_date = doc.get('c_date') + if isinstance(c_date, datetime.datetime): + c_date_str = c_date.strftime('%Y-%m-%d %H:%M:%S') + else: + c_date_str = str(c_date) if c_date is not None else '' + + row = { + '请求时间': c_date_str, + # 输入参数 + '城市': param_map.get('city', ''), + '地址': param_map.get('address', ''), + # 响应参数 + '省份': res_map.get('province', ''), + '省份编码': res_map.get('provinceCode', ''), + '城市名称': res_map.get('city', ''), + '城市编码': res_map.get('cityCode', ''), + '区域': res_map.get('district', ''), + '区域编码': res_map.get('districtCode', ''), + '乡镇街道': res_map.get('town', ''), + '乡镇街道编码': res_map.get('townCode', ''), + '是否涉农': res_map.get('verifyFlag', ''), + '所属区域': res_map.get('addrArea', ''), + '地址验证': res_map.get('addrVerify', ''), + '是否为县级市': res_map.get('countyLevelCity', ''), + '是否县城城区': res_map.get('countyGov', ''), + '是否为省国家级开发区': res_map.get('devDistrict', ''), + '省国家级开发区名称': res_map.get('devDistrictName', ''), + '开发区等级': res_map.get('devLevel', '') + } + rows.append(row) + + total_count = len(rows) + print(f"扫描命中记录: {scanned_count} 条,其中符合对公涉农(type=2)记录: {total_count} 条") + + # 生成 Excel 文件 + tmp_dir = os.path.join(os.path.expanduser('~'), 'tmp', 'njcb_statement') + os.makedirs(tmp_dir, exist_ok=True) + filename = f"南京银行涉农API对账单_{last_year}{last_month:02d}.xlsx" + excel_path = os.path.join(tmp_dir, filename) + + if rows: + df = pd.DataFrame(rows, columns=EXCEL_COLUMNS) + else: + df = pd.DataFrame(columns=EXCEL_COLUMNS) + + df.to_excel(excel_path, index=False, engine='openpyxl') + print(f"Excel文件生成成功: {excel_path}") + + # 发送邮件 + receiver = 'wangwenyan@fangjia.com' + subject = f"南京银行{last_year}年{last_month:02d}月涉农API对账单" + html_content = f""" +

南京银行 顾老师:

+

您好!

+

附件为南京银行 {last_year}年{last_month:02d}月 涉农识别API(/addrSvc/njcbVillageIdentify)对公涉农对账单,请查收。

+ +

如有任何疑问,请随时与我们联系。

+ """ + + print(f"正在发送邮件至: {receiver}, 附件: {excel_path}") + send_email( + to=receiver, + subject=subject, + html_content=html_content, + files=[excel_path] + ) + print("邮件发送完成。") + + +if __name__ == '__main__': + export_and_send_njcb_statement() diff --git a/dags/api/dag_njcb_village_statement.py b/dags/api/dag_njcb_village_statement.py index c3fd720..4576d3f 100644 --- a/dags/api/dag_njcb_village_statement.py +++ b/dags/api/dag_njcb_village_statement.py @@ -1,192 +1,21 @@ -import os -import datetime -import calendar -import json -import pandas as pd -from goojia_utils import LOG_MONGO_CLIENT +# -*- coding: utf-8 -*- from airflow import DAG -from airflow.operators.python import PythonOperator -from airflow.utils.email import send_email +from airflow.providers.standard.operators.bash import BashOperator import pendulum - -def get_last_month_range(ref_date: datetime.date = None): - """ - 获取参考日期的上个月第一天 00:00:00 与最后一天 23:59:59.999999 - 以北京时间为准 - """ - if ref_date is None: - ref_date = datetime.date.today() - - first_day_current_month = ref_date.replace(day=1) - last_day_last_month = first_day_current_month - datetime.timedelta(days=1) - first_day_last_month = last_day_last_month.replace(day=1) - - start_time = datetime.datetime.combine(first_day_last_month, datetime.time.min) - end_time = datetime.datetime.combine(last_day_last_month, datetime.time.max) - - return start_time, end_time, first_day_last_month.year, first_day_last_month.month - - -def parse_dict(val): - """将字符串类型的 JSON 或字典解析为 dict""" - if isinstance(val, dict): - return val - if isinstance(val, str) and val.strip(): - try: - return json.loads(val) - except Exception: - return {} - return {} - - -# Excel 中文表头定义(已剔除:涉农类型、API访问授权凭证、居委、居委编码、城乡代码、城乡代码对应中文) -EXCEL_COLUMNS = [ - '请求时间', - # 输入参数 - '城市', - '地址', - # 响应参数 - '省份', - '省份编码', - '城市名称', - '城市编码', - '区域', - '区域编码', - '乡镇街道', - '乡镇街道编码', - '是否涉农', - '所属区域', - '地址验证', - '是否为县级市', - '是否县城城区', - '是否为省国家级开发区', - '省国家级开发区名称', - '开发区等级' -] - - -def export_and_send_njcb_statement(**context): - """ - 查询上月对公涉农API日志,生成全中文表头Excel对账单并通过邮件发送给南京银行顾老师 - """ - now = datetime.datetime.now() - start_time, end_time, last_year, last_month = get_last_month_range(now.date()) - - print(f"统计时间范围: {start_time} 至 {end_time} (归属年月: {last_year}年{last_month:02d}月)") - - open_uri_log_conn = LOG_MONGO_CLIENT.get_collection('business', 'open_uri_log') - - # 查询条件: - # user_id = 1007801, request_uri = /addrSvc/njcbVillageIdentify, c_date在时间范围内, 且只输出code是200的响应 - query = { - 'user_id': '1007801', - 'request_uri': '/addrSvc/njcbVillageIdentify', - 'c_date': { - '$gte': start_time, - '$lte': end_time - }, - '$or': [ - {'code': 200}, - {'code': '200'} - ] - } - - cursor = open_uri_log_conn.find(query).batch_size(2000) - - rows = [] - for doc in cursor: - param_map = parse_dict(doc.get('parameter_map')) - res_map = parse_dict(doc.get('res_map')) - - # 过滤只保留"对公涉农"的请求 - # 涉农类型: 1 对私,2 对公 (默认为对私) - req_type = str(param_map.get('type', '1')).strip() - if req_type != '2': - continue - - c_date = doc.get('c_date') - if isinstance(c_date, datetime.datetime): - c_date_str = c_date.strftime('%Y-%m-%d %H:%M:%S') - else: - c_date_str = str(c_date) if c_date is not None else '' - - row = { - '请求时间': c_date_str, - # 输入参数 - '城市': param_map.get('city', ''), - '地址': param_map.get('address', ''), - # 响应参数 - '省份': res_map.get('province', ''), - '省份编码': res_map.get('provinceCode', ''), - '城市名称': res_map.get('city', ''), - '城市编码': res_map.get('cityCode', ''), - '区域': res_map.get('district', ''), - '区域编码': res_map.get('districtCode', ''), - '乡镇街道': res_map.get('town', ''), - '乡镇街道编码': res_map.get('townCode', ''), - '是否涉农': res_map.get('verifyFlag', ''), - '所属区域': res_map.get('addrArea', ''), - '地址验证': res_map.get('addrVerify', ''), - '是否为县级市': res_map.get('countyLevelCity', ''), - '是否县城城区': res_map.get('countyGov', ''), - '是否为省国家级开发区': res_map.get('devDistrict', ''), - '省国家级开发区名称': res_map.get('devDistrictName', ''), - '开发区等级': res_map.get('devLevel', '') - } - rows.append(row) - - total_count = len(rows) - print(f"符合条件的对公涉农调用记录共: {total_count} 条") - - # 生成 Excel 文件 - tmp_dir = os.path.join(os.path.expanduser('~'), 'tmp', 'njcb_statement') - os.makedirs(tmp_dir, exist_ok=True) - filename = f"南京银行涉农API对账单_{last_year}{last_month:02d}.xlsx" - excel_path = os.path.join(tmp_dir, filename) - - if rows: - df = pd.DataFrame(rows, columns=EXCEL_COLUMNS) - else: - df = pd.DataFrame(columns=EXCEL_COLUMNS) - - df.to_excel(excel_path, index=False, engine='openpyxl') - - # 发送邮件 - receiver = 'wangwenyan@fangjia.com' - subject = f"南京银行{last_year}年{last_month:02d}月涉农API对账单" - html_content = f""" -

南京银行 顾老师:

-

您好!

-

附件为南京银行 {last_year}年{last_month:02d}月 涉农识别API(/addrSvc/njcbVillageIdentify)对公涉农对账单,请查收。

- -

如有任何疑问,请随时与我们联系。

- """ - - print(f"正在发送邮件至: {receiver}, 附件: {excel_path}") - send_email( - to=receiver, - subject=subject, - html_content=html_content, - files=[excel_path] - ) - print("邮件发送完成。") - - -# Airflow DAG 定义:每月2号上午9:00执行(北京时间) +# 每月2号上午9:00执行(北京时间) with DAG( dag_id="njcb_village_identify_statement_dag", schedule="0 9 2 * *", - start_date=pendulum.datetime(2026, 10, 1, tz="Asia/Shanghai"), + start_date=pendulum.datetime(2024, 1, 1, tz="Asia/Shanghai"), catchup=False, description="每月2号给南京银行顾老师发送上个月涉农API对账单", - tags=["api", "njcb"] + tags=["api", "njcb", "statement"] ) as dag: - send_statement_task = PythonOperator( + send_statement_task = BashOperator( task_id="send_njcb_village_statement", - python_callable=export_and_send_njcb_statement + bash_command="python /opt/airflow/dags/repo/dags/api/api/njcb_village_statement.py" ) + + send_statement_task