refactor(poi): 将县政府乡镇街道核验任务整合为单一DAG文件,避免拆分导致的邮件发送环境问题
This commit is contained in:
@@ -1,214 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
县政府所在乡镇街道核验排查脚本
|
||||
排查流程:
|
||||
1. 查询 192.168.0.206 ES 索引 administrative_mapping_45 中的区域数据(type="区域"),排除 shape 字段。
|
||||
2. 遍历每个区域,获取其 location 经纬度,调用高德逆地理编码 API 获取高德识别的乡镇街道 township。
|
||||
3. 根据民政部区划代码 mzb_district_code 查询该区域下 has_gov="是" 的乡镇街道信息(type="乡镇街道")。
|
||||
4. 对比 ES 中标记为政府所在地的街道名称与高德返回的 township,若不一致则收集。
|
||||
5. 生成排查结果 Excel,发送至 wangwenyan@fangjia.com 供人工审核。
|
||||
"""
|
||||
import os
|
||||
import time
|
||||
import requests
|
||||
import pandas as pd
|
||||
from datetime import datetime
|
||||
from goojia_utils import ElasticSearchClient
|
||||
from airflow.utils.email import send_email
|
||||
|
||||
# 高德 Web 服务 API Key
|
||||
AMAP_KEY = "a34894d151fffff94f9a33bae3d5565c"
|
||||
INDEX_NAME = "administrative_mapping_45"
|
||||
ES_HOST = "192.168.0.206"
|
||||
|
||||
|
||||
def get_amap_township(lon, lat, key=AMAP_KEY, max_retries=3):
|
||||
"""
|
||||
调用高德逆地理编码接口获取 township (乡镇/街道)
|
||||
https://restapi.amap.com/v3/geocode/regeo?location=lon,lat&extensions=base&roadlevel=0
|
||||
"""
|
||||
url = f"https://restapi.amap.com/v3/geocode/regeo"
|
||||
params = {
|
||||
"key": key,
|
||||
"location": f"{lon},{lat}",
|
||||
"extensions": "base",
|
||||
"roadlevel": 0
|
||||
}
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
resp = requests.get(url, params=params, timeout=5)
|
||||
if resp.status_code == 200:
|
||||
data = resp.json()
|
||||
if data.get("status") == "1":
|
||||
regeocode = data.get("regeocode", {})
|
||||
addr_comp = regeocode.get("addressComponent", {})
|
||||
township = addr_comp.get("township")
|
||||
if isinstance(township, str):
|
||||
return township.strip()
|
||||
elif isinstance(township, list) and township:
|
||||
return str(township[0]).strip()
|
||||
return ""
|
||||
else:
|
||||
print(f"高德接口返回异常: {data.get('info')}, code: {data.get('infocode')}")
|
||||
except Exception as e:
|
||||
print(f"请求高德逆地理编码失败 (第{attempt+1}次): {e}")
|
||||
time.sleep(0.5)
|
||||
return ""
|
||||
|
||||
|
||||
def check_gov_township_diff():
|
||||
print(f"[{datetime.now()}] 开始连接 206 ES 集群...")
|
||||
es_client = ElasticSearchClient(host=ES_HOST)
|
||||
|
||||
# 1. 查询 administrative_mapping_45 中的区域数据,排除超大围栏 shape
|
||||
query_district = {
|
||||
"_source": {
|
||||
"excludes": ["shape", "geo_shape", "boundary", "geometry", "polygon"]
|
||||
},
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{"term": {"type": "区域"}}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
print(f"[{datetime.now()}] 正在扫描区域数据 (type='区域')...")
|
||||
district_docs = es_client.scan(index=INDEX_NAME, query=query_district)
|
||||
|
||||
diff_list = []
|
||||
total_districts = 0
|
||||
processed_count = 0
|
||||
|
||||
for hit in district_docs:
|
||||
total_districts += 1
|
||||
source = hit.get("_source", {})
|
||||
doc_id = hit.get("_id")
|
||||
|
||||
province = source.get("province", "")
|
||||
city = source.get("city", "")
|
||||
district = source.get("district", "")
|
||||
mzb_district_code = source.get("mzb_district_code", "")
|
||||
tjj_district_code = source.get("tjj_district_code", "")
|
||||
location = source.get("location", {})
|
||||
|
||||
if not location or "lon" not in location or "lat" not in location:
|
||||
print(f"区域 {province}-{city}-{district}({mzb_district_code}) 缺少有效经纬度,跳过")
|
||||
continue
|
||||
|
||||
lon = location.get("lon")
|
||||
lat = location.get("lat")
|
||||
if not lon or not lat:
|
||||
continue
|
||||
|
||||
processed_count += 1
|
||||
if processed_count % 50 == 0:
|
||||
print(f"已处理 {processed_count} 个区域, 当前发现差异 {len(diff_list)} 条...")
|
||||
|
||||
# 2. 调用高德 API 获取 township
|
||||
amap_township = get_amap_township(lon, lat)
|
||||
|
||||
# 3. 根据民政部区划代码 mzb_district_code 查询该区域下 has_gov='是' 的街道
|
||||
query_street = {
|
||||
"_source": {
|
||||
"excludes": ["shape", "geo_shape", "boundary", "geometry", "polygon"]
|
||||
},
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{"term": {"type": "乡镇街道"}},
|
||||
{"term": {"mzb_district_code": str(mzb_district_code)}},
|
||||
{"term": {"has_gov": "是"}}
|
||||
]
|
||||
}
|
||||
},
|
||||
"from": 0,
|
||||
"size": 10
|
||||
}
|
||||
|
||||
try:
|
||||
res_street = es_client.elastic_client.search(index=INDEX_NAME, body=query_street)
|
||||
street_hits = res_street.get("hits", {}).get("hits", [])
|
||||
except Exception as e:
|
||||
print(f"查询街道数据异常 [{mzb_district_code}]: {e}")
|
||||
street_hits = []
|
||||
|
||||
es_gov_streets = []
|
||||
for s_hit in street_hits:
|
||||
s_source = s_hit.get("_source", {})
|
||||
# street 字段名称可能为 street、name 或 township
|
||||
s_name = s_source.get("street") or s_source.get("name") or s_source.get("township") or ""
|
||||
if s_name:
|
||||
es_gov_streets.append(s_name)
|
||||
|
||||
es_gov_street_str = ",".join(es_gov_streets)
|
||||
|
||||
# 4. 对比 ES 中的街道名称和高德 township
|
||||
# 不一致的情况:没有标记街道、或者标记的街道与高德不匹配
|
||||
is_diff = False
|
||||
if not es_gov_streets:
|
||||
# ES中没有标记 has_gov='是' 的乡镇街道
|
||||
is_diff = True
|
||||
else:
|
||||
# 判断高德 township 是否在标记列表中(考虑后缀差异如镇/街道,或完全不相等)
|
||||
if amap_township not in es_gov_streets:
|
||||
# 简单清洗匹配(如去掉“街道/镇/乡”后是否相同)
|
||||
cleaned_amap = amap_township.replace("街道", "").replace("镇", "").replace("乡", "") if amap_township else ""
|
||||
cleaned_es = [s.replace("街道", "").replace("镇", "").replace("乡", "") for s in es_gov_streets]
|
||||
if not cleaned_amap or cleaned_amap not in cleaned_es:
|
||||
is_diff = True
|
||||
|
||||
if is_diff:
|
||||
diff_list.append({
|
||||
"ES文档ID": doc_id,
|
||||
"省份": province,
|
||||
"城市": city,
|
||||
"区域": district,
|
||||
"民政部区划代码": mzb_district_code,
|
||||
"统计局区划代码": tjj_district_code,
|
||||
"经度(lon)": lon,
|
||||
"纬度(lat)": lat,
|
||||
"ES库中县政府所在街道(has_gov=是)": es_gov_street_str if es_gov_street_str else "【未标记】",
|
||||
"高德逆地理街道(township)": amap_township if amap_township else "【高德未返回】",
|
||||
"排查时间": datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
})
|
||||
|
||||
print(f"[{datetime.now()}] 扫描完成!共遍历区域: {processed_count} 个,发现不一致/缺失记录: {len(diff_list)} 条")
|
||||
|
||||
# 5. 生成 Excel 并邮件发送
|
||||
tmp_dir = os.path.join(os.path.expanduser('~'), 'tmp', 'gov_diff')
|
||||
os.makedirs(tmp_dir, exist_ok=True)
|
||||
today_str = datetime.now().strftime('%Y%m%d_%H%M%S')
|
||||
excel_path = os.path.join(tmp_dir, f"县政府所在乡镇街道差异排查_{today_str}.xlsx")
|
||||
|
||||
df = pd.DataFrame(diff_list)
|
||||
df.to_excel(excel_path, index=False, engine='openpyxl')
|
||||
print(f"排查明细已写入 Excel: {excel_path}")
|
||||
|
||||
# 发送邮件
|
||||
receiver = "wangwenyan@fangjia.com"
|
||||
subject = f"【人工审核】县政府所在乡镇街道差异排查报告 - {datetime.now().strftime('%Y-%m-%d')}"
|
||||
html_content = f"""
|
||||
<h3>王老师:</h3>
|
||||
<p>您好!</p>
|
||||
<p>系统已完成 206 ES <code>{INDEX_NAME}</code> 索引中县政府所在乡镇街道(<code>has_gov='是'</code>)与高德逆地理街道(<code>township</code>)的比对排查,详情如下:</p>
|
||||
<ul>
|
||||
<li><b>总排查区域数:</b> {processed_count} 个</li>
|
||||
<li><b>发现差异/待核对记录数:</b> {len(diff_list)} 条</li>
|
||||
</ul>
|
||||
<p>详细差异名单已附在附件 Excel 中,请人工审核确认。</p>
|
||||
"""
|
||||
|
||||
print(f"正在发送审核邮件至: {receiver} ...")
|
||||
send_email(
|
||||
to=receiver,
|
||||
subject=subject,
|
||||
html_content=html_content,
|
||||
files=[excel_path]
|
||||
)
|
||||
print("邮件发送完成!")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
check_gov_township_diff()
|
||||
@@ -3,14 +3,217 @@
|
||||
县政府所在乡镇街道核验排查定时任务
|
||||
每月1日9点运行,比对 ES administrative_mapping_45 索引中县政府所在街道与高德逆地理街道并发送审核邮件
|
||||
"""
|
||||
import os
|
||||
import time
|
||||
import requests
|
||||
import pandas as pd
|
||||
from datetime import datetime
|
||||
from goojia_utils import ElasticSearchClient
|
||||
from airflow import DAG
|
||||
from airflow.operators.python import PythonOperator
|
||||
from airflow.utils.email import send_email
|
||||
import pendulum
|
||||
from poi.business.check_gov_township_diff import check_gov_township_diff
|
||||
|
||||
# 高德 Web 服务 API Key
|
||||
AMAP_KEY = "a34894d151fffff94f9a33bae3d5565c"
|
||||
INDEX_NAME = "administrative_mapping_45"
|
||||
ES_HOST = "192.168.0.206"
|
||||
|
||||
|
||||
def get_amap_township(lon, lat, key=AMAP_KEY, max_retries=3):
|
||||
"""
|
||||
调用高德逆地理编码接口获取 township (乡镇/街道)
|
||||
https://restapi.amap.com/v3/geocode/regeo?location=lon,lat&extensions=base&roadlevel=0
|
||||
"""
|
||||
url = "https://restapi.amap.com/v3/geocode/regeo"
|
||||
params = {
|
||||
"key": key,
|
||||
"location": f"{lon},{lat}",
|
||||
"extensions": "base",
|
||||
"roadlevel": 0
|
||||
}
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
resp = requests.get(url, params=params, timeout=5)
|
||||
if resp.status_code == 200:
|
||||
data = resp.json()
|
||||
if data.get("status") == "1":
|
||||
regeocode = data.get("regeocode", {})
|
||||
addr_comp = regeocode.get("addressComponent", {})
|
||||
township = addr_comp.get("township")
|
||||
if isinstance(township, str):
|
||||
return township.strip()
|
||||
elif isinstance(township, list) and township:
|
||||
return str(township[0]).strip()
|
||||
return ""
|
||||
else:
|
||||
print(f"高德接口返回异常: {data.get('info')}, code: {data.get('infocode')}")
|
||||
except Exception as e:
|
||||
print(f"请求高德逆地理编码失败 (第{attempt+1}次): {e}")
|
||||
time.sleep(0.5)
|
||||
return ""
|
||||
|
||||
|
||||
def check_gov_township_diff(**context):
|
||||
"""
|
||||
县政府所在乡镇街道比对主任务
|
||||
"""
|
||||
print(f"[{datetime.now()}] 开始连接 206 ES 集群...")
|
||||
es_client = ElasticSearchClient(host=ES_HOST)
|
||||
|
||||
# 1. 查询 administrative_mapping_45 中的区域数据,排除超大围栏 shape
|
||||
query_district = {
|
||||
"_source": {
|
||||
"excludes": ["shape", "geo_shape", "boundary", "geometry", "polygon"]
|
||||
},
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{"term": {"type": "区域"}}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
print(f"[{datetime.now()}] 正在扫描区域数据 (type='区域')...")
|
||||
district_docs = es_client.scan(index=INDEX_NAME, query=query_district)
|
||||
|
||||
diff_list = []
|
||||
total_districts = 0
|
||||
processed_count = 0
|
||||
|
||||
for hit in district_docs:
|
||||
total_districts += 1
|
||||
source = hit.get("_source", {})
|
||||
doc_id = hit.get("_id")
|
||||
|
||||
province = source.get("province", "")
|
||||
city = source.get("city", "")
|
||||
district = source.get("district", "")
|
||||
mzb_district_code = source.get("mzb_district_code", "")
|
||||
tjj_district_code = source.get("tjj_district_code", "")
|
||||
location = source.get("location", {})
|
||||
|
||||
if not location or "lon" not in location or "lat" not in location:
|
||||
print(f"区域 {province}-{city}-{district}({mzb_district_code}) 缺少有效经纬度,跳过")
|
||||
continue
|
||||
|
||||
lon = location.get("lon")
|
||||
lat = location.get("lat")
|
||||
if not lon or not lat:
|
||||
continue
|
||||
|
||||
processed_count += 1
|
||||
if processed_count % 50 == 0:
|
||||
print(f"已处理 {processed_count} 个区域, 当前发现差异 {len(diff_list)} 条...")
|
||||
|
||||
# 2. 调用高德 API 获取 township
|
||||
amap_township = get_amap_township(lon, lat)
|
||||
|
||||
# 3. 根据民政部区划代码 mzb_district_code 查询该区域下 has_gov='是' 的街道
|
||||
query_street = {
|
||||
"_source": {
|
||||
"excludes": ["shape", "geo_shape", "boundary", "geometry", "polygon"]
|
||||
},
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{"term": {"type": "乡镇街道"}},
|
||||
{"term": {"mzb_district_code": str(mzb_district_code)}},
|
||||
{"term": {"has_gov": "是"}}
|
||||
]
|
||||
}
|
||||
},
|
||||
"from": 0,
|
||||
"size": 10
|
||||
}
|
||||
|
||||
try:
|
||||
res_street = es_client.elastic_client.search(index=INDEX_NAME, body=query_street)
|
||||
street_hits = res_street.get("hits", {}).get("hits", [])
|
||||
except Exception as e:
|
||||
print(f"查询街道数据异常 [{mzb_district_code}]: {e}")
|
||||
street_hits = []
|
||||
|
||||
es_gov_streets = []
|
||||
for s_hit in street_hits:
|
||||
s_source = s_hit.get("_source", {})
|
||||
# street 字段名称可能为 street、name 或 township
|
||||
s_name = s_source.get("street") or s_source.get("name") or s_source.get("township") or ""
|
||||
if s_name:
|
||||
es_gov_streets.append(s_name)
|
||||
|
||||
es_gov_street_str = ",".join(es_gov_streets)
|
||||
|
||||
# 4. 对比 ES 中的街道名称和高德 township
|
||||
# 不一致的情况:没有标记街道、或者标记的街道与高德不匹配
|
||||
is_diff = False
|
||||
if not es_gov_streets:
|
||||
# ES中没有标记 has_gov='是' 的乡镇街道
|
||||
is_diff = True
|
||||
else:
|
||||
# 判断高德 township 是否在标记列表中(考虑后缀差异如镇/街道,或完全不相等)
|
||||
if amap_township not in es_gov_streets:
|
||||
# 简单清洗匹配(如去掉“街道/镇/乡”后是否相同)
|
||||
cleaned_amap = amap_township.replace("街道", "").replace("镇", "").replace("乡", "") if amap_township else ""
|
||||
cleaned_es = [s.replace("街道", "").replace("镇", "").replace("乡", "") for s in es_gov_streets]
|
||||
if not cleaned_amap or cleaned_amap not in cleaned_es:
|
||||
is_diff = True
|
||||
|
||||
if is_diff:
|
||||
diff_list.append({
|
||||
"ES文档ID": doc_id,
|
||||
"省份": province,
|
||||
"城市": city,
|
||||
"区域": district,
|
||||
"民政部区划代码": mzb_district_code,
|
||||
"统计局区划代码": tjj_district_code,
|
||||
"经度(lon)": lon,
|
||||
"纬度(lat)": lat,
|
||||
"ES库中县政府所在街道(has_gov=是)": es_gov_street_str if es_gov_street_str else "【未标记】",
|
||||
"高德逆地理街道(township)": amap_township if amap_township else "【高德未返回】",
|
||||
"排查时间": datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
})
|
||||
|
||||
print(f"[{datetime.now()}] 扫描完成!共遍历区域: {processed_count} 个,发现不一致/缺失记录: {len(diff_list)} 条")
|
||||
|
||||
# 5. 生成 Excel 并邮件发送
|
||||
tmp_dir = os.path.join(os.path.expanduser('~'), 'tmp', 'gov_diff')
|
||||
os.makedirs(tmp_dir, exist_ok=True)
|
||||
today_str = datetime.now().strftime('%Y%m%d_%H%M%S')
|
||||
excel_path = os.path.join(tmp_dir, f"县政府所在乡镇街道差异排查_{today_str}.xlsx")
|
||||
|
||||
df = pd.DataFrame(diff_list)
|
||||
df.to_excel(excel_path, index=False, engine='openpyxl')
|
||||
print(f"排查明细已写入 Excel: {excel_path}")
|
||||
|
||||
# 发送邮件
|
||||
receiver = "wangwenyan@fangjia.com"
|
||||
subject = f"【人工审核】县政府所在乡镇街道差异排查报告 - {datetime.now().strftime('%Y-%m-%d')}"
|
||||
html_content = f"""
|
||||
<h3>王老师:</h3>
|
||||
<p>您好!</p>
|
||||
<p>系统已完成 206 ES <code>{INDEX_NAME}</code> 索引中县政府所在乡镇街道(<code>has_gov='是'</code>)与高德逆地理街道(<code>township</code>)的比对排查,详情如下:</p>
|
||||
<ul>
|
||||
<li><b>总排查区域数:</b> {processed_count} 个</li>
|
||||
<li><b>发现差异/待核对记录数:</b> {len(diff_list)} 条</li>
|
||||
</ul>
|
||||
<p>详细差异名单已附在附件 Excel 中,请人工审核确认。</p>
|
||||
"""
|
||||
|
||||
print(f"正在发送审核邮件至: {receiver} ...")
|
||||
send_email(
|
||||
to=receiver,
|
||||
subject=subject,
|
||||
html_content=html_content,
|
||||
files=[excel_path]
|
||||
)
|
||||
print("邮件发送完成!")
|
||||
|
||||
|
||||
# Airflow DAG 定义:每月1号上午9:00执行(北京时间)
|
||||
with DAG(
|
||||
dag_id="check_gov_township_diff_dag",
|
||||
# 每月1日9点运行
|
||||
schedule="0 9 1 * *",
|
||||
start_date=pendulum.datetime(2024, 1, 1, tz="Asia/Shanghai"),
|
||||
catchup=False,
|
||||
|
||||
Reference in New Issue
Block a user