Files
airflow/dags/poi/dag_check_gov_township_diff.py
T

228 lines
8.7 KiB
Python

# -*- coding: utf-8 -*-
"""
县政府所在乡镇街道核验排查定时任务
每月1日9点运行,比对 ES administrative_mapping_45 索引中县政府所在街道与高德逆地理街道并发送审核邮件
"""
import os
import time
import requests
import pandas as pd
from datetime import datetime
from goojia_utils import ElasticSearchClient
from airflow import DAG
from airflow.operators.python import PythonOperator
from airflow.utils.email import send_email
import pendulum
# 高德 Web 服务 API Key
AMAP_KEY = "a34894d151fffff94f9a33bae3d5565c"
INDEX_NAME = "administrative_mapping_45"
ES_HOST = "192.168.0.206"
def get_amap_township(lon, lat, key=AMAP_KEY, max_retries=3):
"""
调用高德逆地理编码接口获取 township (乡镇/街道)
https://restapi.amap.com/v3/geocode/regeo?location=lon,lat&extensions=base&roadlevel=0
"""
url = "https://restapi.amap.com/v3/geocode/regeo"
params = {
"key": key,
"location": f"{lon},{lat}",
"extensions": "base",
"roadlevel": 0
}
for attempt in range(max_retries):
try:
resp = requests.get(url, params=params, timeout=5)
if resp.status_code == 200:
data = resp.json()
if data.get("status") == "1":
regeocode = data.get("regeocode", {})
addr_comp = regeocode.get("addressComponent", {})
township = addr_comp.get("township")
if isinstance(township, str):
return township.strip()
elif isinstance(township, list) and township:
return str(township[0]).strip()
return ""
else:
print(f"高德接口返回异常: {data.get('info')}, code: {data.get('infocode')}")
except Exception as e:
print(f"请求高德逆地理编码失败 (第{attempt+1}次): {e}")
time.sleep(0.5)
return ""
def check_gov_township_diff(**context):
"""
县政府所在乡镇街道比对主任务
"""
print(f"[{datetime.now()}] 开始连接 206 ES 集群...")
es_client = ElasticSearchClient(host=ES_HOST)
# 1. 查询 administrative_mapping_45 中的区域数据,排除超大围栏 shape
query_district = {
"_source": {
"excludes": ["shape", "geo_shape", "boundary", "geometry", "polygon"]
},
"query": {
"bool": {
"must": [
{"term": {"type": "区域"}}
]
}
}
}
print(f"[{datetime.now()}] 正在扫描区域数据 (type='区域')...")
district_docs = es_client.scan(index=INDEX_NAME, query=query_district)
diff_list = []
total_districts = 0
processed_count = 0
for hit in district_docs:
total_districts += 1
source = hit.get("_source", {})
doc_id = hit.get("_id")
province = source.get("province", "")
city = source.get("city", "")
district = source.get("district", "")
mzb_district_code = source.get("mzb_district_code", "")
tjj_district_code = source.get("tjj_district_code", "")
location = source.get("location", {})
if not location or "lon" not in location or "lat" not in location:
print(f"区域 {province}-{city}-{district}({mzb_district_code}) 缺少有效经纬度,跳过")
continue
lon = location.get("lon")
lat = location.get("lat")
if not lon or not lat:
continue
processed_count += 1
if processed_count % 50 == 0:
print(f"已处理 {processed_count} 个区域, 当前发现差异 {len(diff_list)} 条...")
# 2. 调用高德 API 获取 township
amap_township = get_amap_township(lon, lat)
# 3. 根据民政部区划代码 mzb_district_code 查询该区域下 has_gov='是' 的街道
query_street = {
"_source": {
"excludes": ["shape", "geo_shape", "boundary", "geometry", "polygon"]
},
"query": {
"bool": {
"must": [
{"term": {"type": "乡镇街道"}},
{"term": {"mzb_district_code": str(mzb_district_code)}},
{"term": {"has_gov": "是"}}
]
}
},
"from": 0,
"size": 10
}
try:
res_street = es_client.elastic_client.search(index=INDEX_NAME, body=query_street)
street_hits = res_street.get("hits", {}).get("hits", [])
except Exception as e:
print(f"查询街道数据异常 [{mzb_district_code}]: {e}")
street_hits = []
es_gov_streets = []
for s_hit in street_hits:
s_source = s_hit.get("_source", {})
# street 字段名称可能为 street、name 或 township
s_name = s_source.get("street") or s_source.get("name") or s_source.get("township") or ""
if s_name:
es_gov_streets.append(s_name)
es_gov_street_str = ",".join(es_gov_streets)
# 4. 对比 ES 中的街道名称和高德 township
# 不一致的情况:没有标记街道、或者标记的街道与高德不匹配
is_diff = False
if not es_gov_streets:
# ES中没有标记 has_gov='是' 的乡镇街道
is_diff = True
else:
# 判断高德 township 是否在标记列表中(考虑后缀差异如镇/街道,或完全不相等)
if amap_township not in es_gov_streets:
# 简单清洗匹配(如去掉“街道/镇/乡”后是否相同)
cleaned_amap = amap_township.replace("街道", "").replace("镇", "").replace("乡", "") if amap_township else ""
cleaned_es = [s.replace("街道", "").replace("镇", "").replace("乡", "") for s in es_gov_streets]
if not cleaned_amap or cleaned_amap not in cleaned_es:
is_diff = True
if is_diff:
diff_list.append({
"ES文档ID": doc_id,
"省份": province,
"城市": city,
"区域": district,
"民政部区划代码": mzb_district_code,
"统计局区划代码": tjj_district_code,
"经度(lon)": lon,
"纬度(lat)": lat,
"ES库中县政府所在街道(has_gov=是)": es_gov_street_str if es_gov_street_str else "【未标记】",
"高德逆地理街道(township)": amap_township if amap_township else "【高德未返回】",
"排查时间": datetime.now().strftime("%Y-%m-%d %H:%M:%S")
})
print(f"[{datetime.now()}] 扫描完成!共遍历区域: {processed_count} 个,发现不一致/缺失记录: {len(diff_list)} 条")
# 5. 生成 Excel 并邮件发送
tmp_dir = os.path.join(os.path.expanduser('~'), 'tmp', 'gov_diff')
os.makedirs(tmp_dir, exist_ok=True)
today_str = datetime.now().strftime('%Y%m%d_%H%M%S')
excel_path = os.path.join(tmp_dir, f"县政府所在乡镇街道差异排查_{today_str}.xlsx")
df = pd.DataFrame(diff_list)
df.to_excel(excel_path, index=False, engine='openpyxl')
print(f"排查明细已写入 Excel: {excel_path}")
# 发送邮件
receiver = "wangwenyan@fangjia.com"
subject = f"【人工审核】县政府所在乡镇街道差异排查报告 - {datetime.now().strftime('%Y-%m-%d')}"
html_content = f"""
<h3>王老师:</h3>
<p>您好!</p>
<p>系统已完成 206 ES <code>{INDEX_NAME}</code> 索引中县政府所在乡镇街道(<code>has_gov='是'</code>)与高德逆地理街道(<code>township</code>)的比对排查,详情如下:</p>
<ul>
<li><b>总排查区域数:</b> {processed_count} 个</li>
<li><b>发现差异/待核对记录数:</b> {len(diff_list)} 条</li>
</ul>
<p>详细差异名单已附在附件 Excel 中,请人工审核确认。</p>
"""
print(f"正在发送审核邮件至: {receiver} ...")
send_email(
to=receiver,
subject=subject,
html_content=html_content,
files=[excel_path]
)
print("邮件发送完成!")
# Airflow DAG 定义:每月1号上午9:00执行(北京时间)
with DAG(
dag_id="check_gov_township_diff_dag",
schedule="0 9 1 * *",
start_date=pendulum.datetime(2024, 1, 1, tz="Asia/Shanghai"),
catchup=False,
description="县政府所在乡镇街道核验排查(每月1日9点)",
tags=["poi", "gov", "audit"]
) as dag:
check_task = PythonOperator(
task_id="check_gov_township_diff_task",
python_callable=check_gov_township_diff
)