From 6264d35a540b2ce7732b3441bd0f4243fc01ffe4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=96=87=E8=89=B3?= Date: Wed, 16 Sep 2026 09:39:03 +0800 Subject: [PATCH] =?UTF-8?q?refactor(poi):=20=E5=B0=86=E5=8E=BF=E6=94=BF?= =?UTF-8?q?=E5=BA=9C=E4=B9=A1=E9=95=87=E8=A1=97=E9=81=93=E6=A0=B8=E9=AA=8C?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E6=95=B4=E5=90=88=E4=B8=BA=E5=8D=95=E4=B8=80?= =?UTF-8?q?DAG=E6=96=87=E4=BB=B6=EF=BC=8C=E9=81=BF=E5=85=8D=E6=8B=86?= =?UTF-8?q?=E5=88=86=E5=AF=BC=E8=87=B4=E7=9A=84=E9=82=AE=E4=BB=B6=E5=8F=91?= =?UTF-8?q?=E9=80=81=E7=8E=AF=E5=A2=83=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/poi/business/check_gov_township_diff.py | 214 ------------------- dags/poi/dag_check_gov_township_diff.py | 207 +++++++++++++++++- 2 files changed, 205 insertions(+), 216 deletions(-) delete mode 100644 dags/poi/business/check_gov_township_diff.py diff --git a/dags/poi/business/check_gov_township_diff.py b/dags/poi/business/check_gov_township_diff.py deleted file mode 100644 index 14662a8..0000000 --- a/dags/poi/business/check_gov_township_diff.py +++ /dev/null @@ -1,214 +0,0 @@ -# -*- coding: utf-8 -*- -""" -县政府所在乡镇街道核验排查脚本 -排查流程: -1. 查询 192.168.0.206 ES 索引 administrative_mapping_45 中的区域数据(type="区域"),排除 shape 字段。 -2. 遍历每个区域,获取其 location 经纬度,调用高德逆地理编码 API 获取高德识别的乡镇街道 township。 -3. 根据民政部区划代码 mzb_district_code 查询该区域下 has_gov="是" 的乡镇街道信息(type="乡镇街道")。 -4. 对比 ES 中标记为政府所在地的街道名称与高德返回的 township,若不一致则收集。 -5. 生成排查结果 Excel,发送至 wangwenyan@fangjia.com 供人工审核。 -""" -import os -import time -import requests -import pandas as pd -from datetime import datetime -from goojia_utils import ElasticSearchClient -from airflow.utils.email import send_email - -# 高德 Web 服务 API Key -AMAP_KEY = "a34894d151fffff94f9a33bae3d5565c" -INDEX_NAME = "administrative_mapping_45" -ES_HOST = "192.168.0.206" - - -def get_amap_township(lon, lat, key=AMAP_KEY, max_retries=3): - """ - 调用高德逆地理编码接口获取 township (乡镇/街道) - https://restapi.amap.com/v3/geocode/regeo?location=lon,lat&extensions=base&roadlevel=0 - """ - url = f"https://restapi.amap.com/v3/geocode/regeo" - params = { - "key": key, - "location": f"{lon},{lat}", - "extensions": "base", - "roadlevel": 0 - } - for attempt in range(max_retries): - try: - resp = requests.get(url, params=params, timeout=5) - if resp.status_code == 200: - data = resp.json() - if data.get("status") == "1": - regeocode = data.get("regeocode", {}) - addr_comp = regeocode.get("addressComponent", {}) - township = addr_comp.get("township") - if isinstance(township, str): - return township.strip() - elif isinstance(township, list) and township: - return str(township[0]).strip() - return "" - else: - print(f"高德接口返回异常: {data.get('info')}, code: {data.get('infocode')}") - except Exception as e: - print(f"请求高德逆地理编码失败 (第{attempt+1}次): {e}") - time.sleep(0.5) - return "" - - -def check_gov_township_diff(): - print(f"[{datetime.now()}] 开始连接 206 ES 集群...") - es_client = ElasticSearchClient(host=ES_HOST) - - # 1. 查询 administrative_mapping_45 中的区域数据,排除超大围栏 shape - query_district = { - "_source": { - "excludes": ["shape", "geo_shape", "boundary", "geometry", "polygon"] - }, - "query": { - "bool": { - "must": [ - {"term": {"type": "区域"}} - ] - } - } - } - - print(f"[{datetime.now()}] 正在扫描区域数据 (type='区域')...") - district_docs = es_client.scan(index=INDEX_NAME, query=query_district) - - diff_list = [] - total_districts = 0 - processed_count = 0 - - for hit in district_docs: - total_districts += 1 - source = hit.get("_source", {}) - doc_id = hit.get("_id") - - province = source.get("province", "") - city = source.get("city", "") - district = source.get("district", "") - mzb_district_code = source.get("mzb_district_code", "") - tjj_district_code = source.get("tjj_district_code", "") - location = source.get("location", {}) - - if not location or "lon" not in location or "lat" not in location: - print(f"区域 {province}-{city}-{district}({mzb_district_code}) 缺少有效经纬度,跳过") - continue - - lon = location.get("lon") - lat = location.get("lat") - if not lon or not lat: - continue - - processed_count += 1 - if processed_count % 50 == 0: - print(f"已处理 {processed_count} 个区域, 当前发现差异 {len(diff_list)} 条...") - - # 2. 调用高德 API 获取 township - amap_township = get_amap_township(lon, lat) - - # 3. 根据民政部区划代码 mzb_district_code 查询该区域下 has_gov='是' 的街道 - query_street = { - "_source": { - "excludes": ["shape", "geo_shape", "boundary", "geometry", "polygon"] - }, - "query": { - "bool": { - "must": [ - {"term": {"type": "乡镇街道"}}, - {"term": {"mzb_district_code": str(mzb_district_code)}}, - {"term": {"has_gov": "是"}} - ] - } - }, - "from": 0, - "size": 10 - } - - try: - res_street = es_client.elastic_client.search(index=INDEX_NAME, body=query_street) - street_hits = res_street.get("hits", {}).get("hits", []) - except Exception as e: - print(f"查询街道数据异常 [{mzb_district_code}]: {e}") - street_hits = [] - - es_gov_streets = [] - for s_hit in street_hits: - s_source = s_hit.get("_source", {}) - # street 字段名称可能为 street、name 或 township - s_name = s_source.get("street") or s_source.get("name") or s_source.get("township") or "" - if s_name: - es_gov_streets.append(s_name) - - es_gov_street_str = ",".join(es_gov_streets) - - # 4. 对比 ES 中的街道名称和高德 township - # 不一致的情况:没有标记街道、或者标记的街道与高德不匹配 - is_diff = False - if not es_gov_streets: - # ES中没有标记 has_gov='是' 的乡镇街道 - is_diff = True - else: - # 判断高德 township 是否在标记列表中(考虑后缀差异如镇/街道,或完全不相等) - if amap_township not in es_gov_streets: - # 简单清洗匹配(如去掉“街道/镇/乡”后是否相同) - cleaned_amap = amap_township.replace("街道", "").replace("镇", "").replace("乡", "") if amap_township else "" - cleaned_es = [s.replace("街道", "").replace("镇", "").replace("乡", "") for s in es_gov_streets] - if not cleaned_amap or cleaned_amap not in cleaned_es: - is_diff = True - - if is_diff: - diff_list.append({ - "ES文档ID": doc_id, - "省份": province, - "城市": city, - "区域": district, - "民政部区划代码": mzb_district_code, - "统计局区划代码": tjj_district_code, - "经度(lon)": lon, - "纬度(lat)": lat, - "ES库中县政府所在街道(has_gov=是)": es_gov_street_str if es_gov_street_str else "【未标记】", - "高德逆地理街道(township)": amap_township if amap_township else "【高德未返回】", - "排查时间": datetime.now().strftime("%Y-%m-%d %H:%M:%S") - }) - - print(f"[{datetime.now()}] 扫描完成!共遍历区域: {processed_count} 个,发现不一致/缺失记录: {len(diff_list)} 条") - - # 5. 生成 Excel 并邮件发送 - tmp_dir = os.path.join(os.path.expanduser('~'), 'tmp', 'gov_diff') - os.makedirs(tmp_dir, exist_ok=True) - today_str = datetime.now().strftime('%Y%m%d_%H%M%S') - excel_path = os.path.join(tmp_dir, f"县政府所在乡镇街道差异排查_{today_str}.xlsx") - - df = pd.DataFrame(diff_list) - df.to_excel(excel_path, index=False, engine='openpyxl') - print(f"排查明细已写入 Excel: {excel_path}") - - # 发送邮件 - receiver = "wangwenyan@fangjia.com" - subject = f"【人工审核】县政府所在乡镇街道差异排查报告 - {datetime.now().strftime('%Y-%m-%d')}" - html_content = f""" -

王老师:

-

您好!

-

系统已完成 206 ES {INDEX_NAME} 索引中县政府所在乡镇街道(has_gov='是')与高德逆地理街道(township)的比对排查,详情如下:

- -

详细差异名单已附在附件 Excel 中,请人工审核确认。

- """ - - print(f"正在发送审核邮件至: {receiver} ...") - send_email( - to=receiver, - subject=subject, - html_content=html_content, - files=[excel_path] - ) - print("邮件发送完成!") - - -if __name__ == '__main__': - check_gov_township_diff() diff --git a/dags/poi/dag_check_gov_township_diff.py b/dags/poi/dag_check_gov_township_diff.py index 9edea23..e7bf48b 100644 --- a/dags/poi/dag_check_gov_township_diff.py +++ b/dags/poi/dag_check_gov_township_diff.py @@ -3,14 +3,217 @@ 县政府所在乡镇街道核验排查定时任务 每月1日9点运行,比对 ES administrative_mapping_45 索引中县政府所在街道与高德逆地理街道并发送审核邮件 """ +import os +import time +import requests +import pandas as pd +from datetime import datetime +from goojia_utils import ElasticSearchClient from airflow import DAG from airflow.operators.python import PythonOperator +from airflow.utils.email import send_email import pendulum -from poi.business.check_gov_township_diff import check_gov_township_diff +# 高德 Web 服务 API Key +AMAP_KEY = "a34894d151fffff94f9a33bae3d5565c" +INDEX_NAME = "administrative_mapping_45" +ES_HOST = "192.168.0.206" + + +def get_amap_township(lon, lat, key=AMAP_KEY, max_retries=3): + """ + 调用高德逆地理编码接口获取 township (乡镇/街道) + https://restapi.amap.com/v3/geocode/regeo?location=lon,lat&extensions=base&roadlevel=0 + """ + url = "https://restapi.amap.com/v3/geocode/regeo" + params = { + "key": key, + "location": f"{lon},{lat}", + "extensions": "base", + "roadlevel": 0 + } + for attempt in range(max_retries): + try: + resp = requests.get(url, params=params, timeout=5) + if resp.status_code == 200: + data = resp.json() + if data.get("status") == "1": + regeocode = data.get("regeocode", {}) + addr_comp = regeocode.get("addressComponent", {}) + township = addr_comp.get("township") + if isinstance(township, str): + return township.strip() + elif isinstance(township, list) and township: + return str(township[0]).strip() + return "" + else: + print(f"高德接口返回异常: {data.get('info')}, code: {data.get('infocode')}") + except Exception as e: + print(f"请求高德逆地理编码失败 (第{attempt+1}次): {e}") + time.sleep(0.5) + return "" + + +def check_gov_township_diff(**context): + """ + 县政府所在乡镇街道比对主任务 + """ + print(f"[{datetime.now()}] 开始连接 206 ES 集群...") + es_client = ElasticSearchClient(host=ES_HOST) + + # 1. 查询 administrative_mapping_45 中的区域数据,排除超大围栏 shape + query_district = { + "_source": { + "excludes": ["shape", "geo_shape", "boundary", "geometry", "polygon"] + }, + "query": { + "bool": { + "must": [ + {"term": {"type": "区域"}} + ] + } + } + } + + print(f"[{datetime.now()}] 正在扫描区域数据 (type='区域')...") + district_docs = es_client.scan(index=INDEX_NAME, query=query_district) + + diff_list = [] + total_districts = 0 + processed_count = 0 + + for hit in district_docs: + total_districts += 1 + source = hit.get("_source", {}) + doc_id = hit.get("_id") + + province = source.get("province", "") + city = source.get("city", "") + district = source.get("district", "") + mzb_district_code = source.get("mzb_district_code", "") + tjj_district_code = source.get("tjj_district_code", "") + location = source.get("location", {}) + + if not location or "lon" not in location or "lat" not in location: + print(f"区域 {province}-{city}-{district}({mzb_district_code}) 缺少有效经纬度,跳过") + continue + + lon = location.get("lon") + lat = location.get("lat") + if not lon or not lat: + continue + + processed_count += 1 + if processed_count % 50 == 0: + print(f"已处理 {processed_count} 个区域, 当前发现差异 {len(diff_list)} 条...") + + # 2. 调用高德 API 获取 township + amap_township = get_amap_township(lon, lat) + + # 3. 根据民政部区划代码 mzb_district_code 查询该区域下 has_gov='是' 的街道 + query_street = { + "_source": { + "excludes": ["shape", "geo_shape", "boundary", "geometry", "polygon"] + }, + "query": { + "bool": { + "must": [ + {"term": {"type": "乡镇街道"}}, + {"term": {"mzb_district_code": str(mzb_district_code)}}, + {"term": {"has_gov": "是"}} + ] + } + }, + "from": 0, + "size": 10 + } + + try: + res_street = es_client.elastic_client.search(index=INDEX_NAME, body=query_street) + street_hits = res_street.get("hits", {}).get("hits", []) + except Exception as e: + print(f"查询街道数据异常 [{mzb_district_code}]: {e}") + street_hits = [] + + es_gov_streets = [] + for s_hit in street_hits: + s_source = s_hit.get("_source", {}) + # street 字段名称可能为 street、name 或 township + s_name = s_source.get("street") or s_source.get("name") or s_source.get("township") or "" + if s_name: + es_gov_streets.append(s_name) + + es_gov_street_str = ",".join(es_gov_streets) + + # 4. 对比 ES 中的街道名称和高德 township + # 不一致的情况:没有标记街道、或者标记的街道与高德不匹配 + is_diff = False + if not es_gov_streets: + # ES中没有标记 has_gov='是' 的乡镇街道 + is_diff = True + else: + # 判断高德 township 是否在标记列表中(考虑后缀差异如镇/街道,或完全不相等) + if amap_township not in es_gov_streets: + # 简单清洗匹配(如去掉“街道/镇/乡”后是否相同) + cleaned_amap = amap_township.replace("街道", "").replace("镇", "").replace("乡", "") if amap_township else "" + cleaned_es = [s.replace("街道", "").replace("镇", "").replace("乡", "") for s in es_gov_streets] + if not cleaned_amap or cleaned_amap not in cleaned_es: + is_diff = True + + if is_diff: + diff_list.append({ + "ES文档ID": doc_id, + "省份": province, + "城市": city, + "区域": district, + "民政部区划代码": mzb_district_code, + "统计局区划代码": tjj_district_code, + "经度(lon)": lon, + "纬度(lat)": lat, + "ES库中县政府所在街道(has_gov=是)": es_gov_street_str if es_gov_street_str else "【未标记】", + "高德逆地理街道(township)": amap_township if amap_township else "【高德未返回】", + "排查时间": datetime.now().strftime("%Y-%m-%d %H:%M:%S") + }) + + print(f"[{datetime.now()}] 扫描完成!共遍历区域: {processed_count} 个,发现不一致/缺失记录: {len(diff_list)} 条") + + # 5. 生成 Excel 并邮件发送 + tmp_dir = os.path.join(os.path.expanduser('~'), 'tmp', 'gov_diff') + os.makedirs(tmp_dir, exist_ok=True) + today_str = datetime.now().strftime('%Y%m%d_%H%M%S') + excel_path = os.path.join(tmp_dir, f"县政府所在乡镇街道差异排查_{today_str}.xlsx") + + df = pd.DataFrame(diff_list) + df.to_excel(excel_path, index=False, engine='openpyxl') + print(f"排查明细已写入 Excel: {excel_path}") + + # 发送邮件 + receiver = "wangwenyan@fangjia.com" + subject = f"【人工审核】县政府所在乡镇街道差异排查报告 - {datetime.now().strftime('%Y-%m-%d')}" + html_content = f""" +

王老师:

+

您好!

+

系统已完成 206 ES {INDEX_NAME} 索引中县政府所在乡镇街道(has_gov='是')与高德逆地理街道(township)的比对排查,详情如下:

+ +

详细差异名单已附在附件 Excel 中,请人工审核确认。

+ """ + + print(f"正在发送审核邮件至: {receiver} ...") + send_email( + to=receiver, + subject=subject, + html_content=html_content, + files=[excel_path] + ) + print("邮件发送完成!") + + +# Airflow DAG 定义:每月1号上午9:00执行(北京时间) with DAG( dag_id="check_gov_township_diff_dag", - # 每月1日9点运行 schedule="0 9 1 * *", start_date=pendulum.datetime(2024, 1, 1, tz="Asia/Shanghai"), catchup=False,