diff --git a/dags/poi/business/check_gov_township_diff.py b/dags/poi/business/check_gov_township_diff.py new file mode 100644 index 0000000..14662a8 --- /dev/null +++ b/dags/poi/business/check_gov_township_diff.py @@ -0,0 +1,214 @@ +# -*- coding: utf-8 -*- +""" +县政府所在乡镇街道核验排查脚本 +排查流程: +1. 查询 192.168.0.206 ES 索引 administrative_mapping_45 中的区域数据(type="区域"),排除 shape 字段。 +2. 遍历每个区域,获取其 location 经纬度,调用高德逆地理编码 API 获取高德识别的乡镇街道 township。 +3. 根据民政部区划代码 mzb_district_code 查询该区域下 has_gov="是" 的乡镇街道信息(type="乡镇街道")。 +4. 对比 ES 中标记为政府所在地的街道名称与高德返回的 township,若不一致则收集。 +5. 生成排查结果 Excel,发送至 wangwenyan@fangjia.com 供人工审核。 +""" +import os +import time +import requests +import pandas as pd +from datetime import datetime +from goojia_utils import ElasticSearchClient +from airflow.utils.email import send_email + +# 高德 Web 服务 API Key +AMAP_KEY = "a34894d151fffff94f9a33bae3d5565c" +INDEX_NAME = "administrative_mapping_45" +ES_HOST = "192.168.0.206" + + +def get_amap_township(lon, lat, key=AMAP_KEY, max_retries=3): + """ + 调用高德逆地理编码接口获取 township (乡镇/街道) + https://restapi.amap.com/v3/geocode/regeo?location=lon,lat&extensions=base&roadlevel=0 + """ + url = f"https://restapi.amap.com/v3/geocode/regeo" + params = { + "key": key, + "location": f"{lon},{lat}", + "extensions": "base", + "roadlevel": 0 + } + for attempt in range(max_retries): + try: + resp = requests.get(url, params=params, timeout=5) + if resp.status_code == 200: + data = resp.json() + if data.get("status") == "1": + regeocode = data.get("regeocode", {}) + addr_comp = regeocode.get("addressComponent", {}) + township = addr_comp.get("township") + if isinstance(township, str): + return township.strip() + elif isinstance(township, list) and township: + return str(township[0]).strip() + return "" + else: + print(f"高德接口返回异常: {data.get('info')}, code: {data.get('infocode')}") + except Exception as e: + print(f"请求高德逆地理编码失败 (第{attempt+1}次): {e}") + time.sleep(0.5) + return "" + + +def check_gov_township_diff(): + print(f"[{datetime.now()}] 开始连接 206 ES 集群...") + es_client = ElasticSearchClient(host=ES_HOST) + + # 1. 查询 administrative_mapping_45 中的区域数据,排除超大围栏 shape + query_district = { + "_source": { + "excludes": ["shape", "geo_shape", "boundary", "geometry", "polygon"] + }, + "query": { + "bool": { + "must": [ + {"term": {"type": "区域"}} + ] + } + } + } + + print(f"[{datetime.now()}] 正在扫描区域数据 (type='区域')...") + district_docs = es_client.scan(index=INDEX_NAME, query=query_district) + + diff_list = [] + total_districts = 0 + processed_count = 0 + + for hit in district_docs: + total_districts += 1 + source = hit.get("_source", {}) + doc_id = hit.get("_id") + + province = source.get("province", "") + city = source.get("city", "") + district = source.get("district", "") + mzb_district_code = source.get("mzb_district_code", "") + tjj_district_code = source.get("tjj_district_code", "") + location = source.get("location", {}) + + if not location or "lon" not in location or "lat" not in location: + print(f"区域 {province}-{city}-{district}({mzb_district_code}) 缺少有效经纬度,跳过") + continue + + lon = location.get("lon") + lat = location.get("lat") + if not lon or not lat: + continue + + processed_count += 1 + if processed_count % 50 == 0: + print(f"已处理 {processed_count} 个区域, 当前发现差异 {len(diff_list)} 条...") + + # 2. 调用高德 API 获取 township + amap_township = get_amap_township(lon, lat) + + # 3. 根据民政部区划代码 mzb_district_code 查询该区域下 has_gov='是' 的街道 + query_street = { + "_source": { + "excludes": ["shape", "geo_shape", "boundary", "geometry", "polygon"] + }, + "query": { + "bool": { + "must": [ + {"term": {"type": "乡镇街道"}}, + {"term": {"mzb_district_code": str(mzb_district_code)}}, + {"term": {"has_gov": "是"}} + ] + } + }, + "from": 0, + "size": 10 + } + + try: + res_street = es_client.elastic_client.search(index=INDEX_NAME, body=query_street) + street_hits = res_street.get("hits", {}).get("hits", []) + except Exception as e: + print(f"查询街道数据异常 [{mzb_district_code}]: {e}") + street_hits = [] + + es_gov_streets = [] + for s_hit in street_hits: + s_source = s_hit.get("_source", {}) + # street 字段名称可能为 street、name 或 township + s_name = s_source.get("street") or s_source.get("name") or s_source.get("township") or "" + if s_name: + es_gov_streets.append(s_name) + + es_gov_street_str = ",".join(es_gov_streets) + + # 4. 对比 ES 中的街道名称和高德 township + # 不一致的情况:没有标记街道、或者标记的街道与高德不匹配 + is_diff = False + if not es_gov_streets: + # ES中没有标记 has_gov='是' 的乡镇街道 + is_diff = True + else: + # 判断高德 township 是否在标记列表中(考虑后缀差异如镇/街道,或完全不相等) + if amap_township not in es_gov_streets: + # 简单清洗匹配(如去掉“街道/镇/乡”后是否相同) + cleaned_amap = amap_township.replace("街道", "").replace("镇", "").replace("乡", "") if amap_township else "" + cleaned_es = [s.replace("街道", "").replace("镇", "").replace("乡", "") for s in es_gov_streets] + if not cleaned_amap or cleaned_amap not in cleaned_es: + is_diff = True + + if is_diff: + diff_list.append({ + "ES文档ID": doc_id, + "省份": province, + "城市": city, + "区域": district, + "民政部区划代码": mzb_district_code, + "统计局区划代码": tjj_district_code, + "经度(lon)": lon, + "纬度(lat)": lat, + "ES库中县政府所在街道(has_gov=是)": es_gov_street_str if es_gov_street_str else "【未标记】", + "高德逆地理街道(township)": amap_township if amap_township else "【高德未返回】", + "排查时间": datetime.now().strftime("%Y-%m-%d %H:%M:%S") + }) + + print(f"[{datetime.now()}] 扫描完成!共遍历区域: {processed_count} 个,发现不一致/缺失记录: {len(diff_list)} 条") + + # 5. 生成 Excel 并邮件发送 + tmp_dir = os.path.join(os.path.expanduser('~'), 'tmp', 'gov_diff') + os.makedirs(tmp_dir, exist_ok=True) + today_str = datetime.now().strftime('%Y%m%d_%H%M%S') + excel_path = os.path.join(tmp_dir, f"县政府所在乡镇街道差异排查_{today_str}.xlsx") + + df = pd.DataFrame(diff_list) + df.to_excel(excel_path, index=False, engine='openpyxl') + print(f"排查明细已写入 Excel: {excel_path}") + + # 发送邮件 + receiver = "wangwenyan@fangjia.com" + subject = f"【人工审核】县政府所在乡镇街道差异排查报告 - {datetime.now().strftime('%Y-%m-%d')}" + html_content = f""" +

王老师:

+

您好!

+

系统已完成 206 ES {INDEX_NAME} 索引中县政府所在乡镇街道(has_gov='是')与高德逆地理街道(township)的比对排查,详情如下:

+ +

详细差异名单已附在附件 Excel 中,请人工审核确认。

+ """ + + print(f"正在发送审核邮件至: {receiver} ...") + send_email( + to=receiver, + subject=subject, + html_content=html_content, + files=[excel_path] + ) + print("邮件发送完成!") + + +if __name__ == '__main__': + check_gov_township_diff() diff --git a/dags/poi/dag_check_gov_township_diff.py b/dags/poi/dag_check_gov_township_diff.py new file mode 100644 index 0000000..9edea23 --- /dev/null +++ b/dags/poi/dag_check_gov_township_diff.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +""" +县政府所在乡镇街道核验排查定时任务 +每月1日9点运行,比对 ES administrative_mapping_45 索引中县政府所在街道与高德逆地理街道并发送审核邮件 +""" +from airflow import DAG +from airflow.operators.python import PythonOperator +import pendulum +from poi.business.check_gov_township_diff import check_gov_township_diff + +with DAG( + dag_id="check_gov_township_diff_dag", + # 每月1日9点运行 + schedule="0 9 1 * *", + start_date=pendulum.datetime(2024, 1, 1, tz="Asia/Shanghai"), + catchup=False, + description="县政府所在乡镇街道核验排查(每月1日9点)", + tags=["poi", "gov", "audit"] +) as dag: + + check_task = PythonOperator( + task_id="check_gov_township_diff_task", + python_callable=check_gov_township_diff + )