240 lines
9.3 KiB
Python
240 lines
9.3 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""
|
||
县政府所在乡镇街道核验排查定时任务
|
||
每月1日9点运行,比对 ES administrative_mapping_45 索引中县政府所在街道与高德逆地理街道并发送审核邮件
|
||
"""
|
||
import os
|
||
import time
|
||
import requests
|
||
import pandas as pd
|
||
from datetime import datetime
|
||
from goojia_utils import ElasticSearchClient
|
||
from airflow import DAG
|
||
from airflow.operators.python import PythonOperator
|
||
from airflow.utils.email import send_email
|
||
import pendulum
|
||
|
||
# 高德 Web 服务 API Key
|
||
AMAP_KEY = "a34894d151fffff94f9a33bae3d5565c"
|
||
INDEX_NAME = "administrative_mapping_45"
|
||
ES_HOST = "192.168.0.206"
|
||
|
||
|
||
def get_amap_township(lon, lat, key=AMAP_KEY, max_retries=3):
|
||
"""
|
||
调用高德逆地理编码接口获取 township (乡镇/街道)
|
||
https://restapi.amap.com/v3/geocode/regeo?location=lon,lat&extensions=base&roadlevel=0
|
||
"""
|
||
url = "https://restapi.amap.com/v3/geocode/regeo"
|
||
params = {
|
||
"key": key,
|
||
"location": f"{lon},{lat}",
|
||
"extensions": "base",
|
||
"roadlevel": 0
|
||
}
|
||
for attempt in range(max_retries):
|
||
try:
|
||
resp = requests.get(url, params=params, timeout=5)
|
||
if resp.status_code == 200:
|
||
data = resp.json()
|
||
if data.get("status") == "1":
|
||
regeocode = data.get("regeocode", {})
|
||
addr_comp = regeocode.get("addressComponent", {})
|
||
township = addr_comp.get("township")
|
||
if isinstance(township, str):
|
||
return township.strip()
|
||
elif isinstance(township, list) and township:
|
||
return str(township[0]).strip()
|
||
return ""
|
||
else:
|
||
print(f"高德接口返回异常: {data.get('info')}, code: {data.get('infocode')}")
|
||
except Exception as e:
|
||
print(f"请求高德逆地理编码失败 (第{attempt+1}次): {e}")
|
||
time.sleep(0.5)
|
||
return ""
|
||
|
||
|
||
def check_gov_township_diff(**context):
|
||
"""
|
||
县政府所在乡镇街道比对主任务,支持从 dag_run.conf 或 params 中传入自定义 amap_key
|
||
"""
|
||
dag_run = context.get('dag_run')
|
||
conf = (dag_run.conf if dag_run else {}) or {}
|
||
params = context.get('params') or {}
|
||
amap_key = conf.get('amap_key') or conf.get('key') or params.get('amap_key') or AMAP_KEY
|
||
amap_key = str(amap_key).strip()
|
||
|
||
print(f"[{datetime.now()}] 使用的高德 API Key: {amap_key[:6]}******{amap_key[-4:] if len(amap_key) > 10 else ''}")
|
||
print(f"[{datetime.now()}] 开始连接 206 ES 集群...")
|
||
es_client = ElasticSearchClient(host=ES_HOST)
|
||
|
||
# 1. 查询 administrative_mapping_45 中的区域数据,排除超大围栏 shape
|
||
query_district = {
|
||
"_source": {
|
||
"excludes": ["shape", "geo_shape", "boundary", "geometry", "polygon"]
|
||
},
|
||
"query": {
|
||
"bool": {
|
||
"must": [
|
||
{"term": {"type": "区域"}}
|
||
]
|
||
}
|
||
}
|
||
}
|
||
|
||
print(f"[{datetime.now()}] 正在扫描区域数据 (type='区域')...")
|
||
district_docs = es_client.scan(index=INDEX_NAME, query=query_district)
|
||
|
||
diff_list = []
|
||
total_districts = 0
|
||
processed_count = 0
|
||
|
||
for hit in district_docs:
|
||
total_districts += 1
|
||
source = hit.get("_source", {})
|
||
doc_id = hit.get("_id")
|
||
|
||
province = source.get("province", "")
|
||
city = source.get("city", "")
|
||
district = source.get("district", "")
|
||
mzb_district_code = source.get("mzb_district_code", "")
|
||
tjj_district_code = source.get("tjj_district_code", "")
|
||
location = source.get("location", {})
|
||
|
||
if not location or "lon" not in location or "lat" not in location:
|
||
print(f"区域 {province}-{city}-{district}({mzb_district_code}) 缺少有效经纬度,跳过")
|
||
continue
|
||
|
||
lon = location.get("lon")
|
||
lat = location.get("lat")
|
||
if not lon or not lat:
|
||
continue
|
||
|
||
processed_count += 1
|
||
if processed_count % 50 == 0:
|
||
print(f"已处理 {processed_count} 个区域, 当前发现差异 {len(diff_list)} 条...")
|
||
|
||
# 2. 调用高德 API 获取 township(传入可配置的 amap_key)
|
||
amap_township = get_amap_township(lon, lat, key=amap_key)
|
||
|
||
# 3. 根据民政部区划代码 mzb_district_code 查询该区域下 has_gov='是' 的街道
|
||
query_street = {
|
||
"_source": {
|
||
"excludes": ["shape", "geo_shape", "boundary", "geometry", "polygon"]
|
||
},
|
||
"query": {
|
||
"bool": {
|
||
"must": [
|
||
{"term": {"type": "乡镇街道"}},
|
||
{"term": {"mzb_district_code": str(mzb_district_code)}},
|
||
{"term": {"has_gov": "是"}}
|
||
]
|
||
}
|
||
},
|
||
"from": 0,
|
||
"size": 10
|
||
}
|
||
|
||
try:
|
||
res_street = es_client.elastic_client.search(index=INDEX_NAME, body=query_street)
|
||
street_hits = res_street.get("hits", {}).get("hits", [])
|
||
except Exception as e:
|
||
print(f"查询街道数据异常 [{mzb_district_code}]: {e}")
|
||
street_hits = []
|
||
|
||
es_gov_streets = []
|
||
for s_hit in street_hits:
|
||
s_source = s_hit.get("_source", {})
|
||
# street 字段名称可能为 street、name 或 township
|
||
s_name = s_source.get("street") or s_source.get("name") or s_source.get("township") or ""
|
||
if s_name:
|
||
es_gov_streets.append(s_name)
|
||
|
||
es_gov_street_str = ",".join(es_gov_streets)
|
||
|
||
# 4. 对比 ES 中的街道名称和高德 township
|
||
# 不一致的情况:没有标记街道、或者标记的街道与高德不匹配
|
||
is_diff = False
|
||
if not es_gov_streets:
|
||
# ES中没有标记 has_gov='是' 的乡镇街道
|
||
is_diff = True
|
||
else:
|
||
# 判断高德 township 是否在标记列表中(考虑后缀差异如镇/街道,或完全不相等)
|
||
if amap_township not in es_gov_streets:
|
||
# 简单清洗匹配(如去掉“街道/镇/乡”后是否相同)
|
||
cleaned_amap = amap_township.replace("街道", "").replace("镇", "").replace("乡", "") if amap_township else ""
|
||
cleaned_es = [s.replace("街道", "").replace("镇", "").replace("乡", "") for s in es_gov_streets]
|
||
if not cleaned_amap or cleaned_amap not in cleaned_es:
|
||
is_diff = True
|
||
|
||
if is_diff:
|
||
diff_list.append({
|
||
"ES文档ID": doc_id,
|
||
"省份": province,
|
||
"城市": city,
|
||
"区域": district,
|
||
"民政部区划代码": mzb_district_code,
|
||
"统计局区划代码": tjj_district_code,
|
||
"经度(lon)": lon,
|
||
"纬度(lat)": lat,
|
||
"ES库中县政府所在街道(has_gov=是)": es_gov_street_str if es_gov_street_str else "【未标记】",
|
||
"高德逆地理街道(township)": amap_township if amap_township else "【高德未返回】",
|
||
"排查时间": datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||
})
|
||
|
||
print(f"[{datetime.now()}] 扫描完成!共遍历区域: {processed_count} 个,发现不一致/缺失记录: {len(diff_list)} 条")
|
||
|
||
# 5. 生成 Excel 并邮件发送
|
||
tmp_dir = os.path.join(os.path.expanduser('~'), 'tmp', 'gov_diff')
|
||
os.makedirs(tmp_dir, exist_ok=True)
|
||
today_str = datetime.now().strftime('%Y%m%d_%H%M%S')
|
||
excel_path = os.path.join(tmp_dir, f"县政府所在乡镇街道差异排查_{today_str}.xlsx")
|
||
|
||
df = pd.DataFrame(diff_list)
|
||
df.to_excel(excel_path, index=False, engine='openpyxl')
|
||
print(f"排查明细已写入 Excel: {excel_path}")
|
||
|
||
# 发送邮件
|
||
receiver = "wangwenyan@fangjia.com"
|
||
cc_receiver = "daijinshui@fangjia.com"
|
||
subject = f"【人工审核】县政府所在乡镇街道差异排查报告 - {datetime.now().strftime('%Y-%m-%d')}"
|
||
html_content = f"""
|
||
<h3>王老师:</h3>
|
||
<p>您好!</p>
|
||
<p>系统已完成 206 ES <code>{INDEX_NAME}</code> 索引中县政府所在乡镇街道(<code>has_gov='是'</code>)与高德逆地理街道(<code>township</code>)的比对排查,详情如下:</p>
|
||
<ul>
|
||
<li><b>总排查区域数:</b> {processed_count} 个</li>
|
||
<li><b>发现差异/待核对记录数:</b> {len(diff_list)} 条</li>
|
||
</ul>
|
||
<p>详细差异名单已附在附件 Excel 中,请人工审核确认。</p>
|
||
"""
|
||
|
||
print(f"正在发送审核邮件至: {receiver}, 抄送: {cc_receiver} ...")
|
||
send_email(
|
||
to=receiver,
|
||
subject=subject,
|
||
html_content=html_content,
|
||
files=[excel_path],
|
||
cc=cc_receiver
|
||
)
|
||
print("邮件发送完成!")
|
||
|
||
|
||
# Airflow DAG 定义:每月1号上午9:00执行(北京时间)
|
||
with DAG(
|
||
dag_id="check_gov_township_diff_dag",
|
||
schedule="0 9 1 * *",
|
||
start_date=pendulum.datetime(2024, 1, 1, tz="Asia/Shanghai"),
|
||
catchup=False,
|
||
description="县政府所在乡镇街道核验排查(每月1日9点)",
|
||
params={
|
||
"amap_key": AMAP_KEY
|
||
},
|
||
tags=["poi", "gov", "audit"]
|
||
) as dag:
|
||
|
||
check_task = PythonOperator(
|
||
task_id="check_gov_township_diff_task",
|
||
python_callable=check_gov_township_diff
|
||
)
|