From 4721f2382755d840455f712c00a4e96035235046 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=96=87=E8=89=B3?= Date: Wed, 22 Jul 2026 16:29:13 +0800 Subject: [PATCH] =?UTF-8?q?=E7=89=88=E6=9C=AC=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/company/company/bd2gaode.py | 140 +++++++++++++++++-------------- 1 file changed, 77 insertions(+), 63 deletions(-) diff --git a/dags/company/company/bd2gaode.py b/dags/company/company/bd2gaode.py index a9ffed2..a753d95 100644 --- a/dags/company/company/bd2gaode.py +++ b/dags/company/company/bd2gaode.py @@ -1,80 +1,94 @@ """ 第一步: 将百度经纬度转为高德经纬度(只需跑批一次即可) TODO """ +import os +# 1. 尝试环境变量兼容 +os.environ["ELASTIC_CLIENT_APIVERSIONING"] = "0" + from goojia_utils import DIDATA_DEV_ELASTIC_CLIENT, threads, geohashutils, areautils import pandas as pd from elasticsearch import helpers import logging + log = logging.getLogger(__name__) -import os -# 🛡️ 极其重要:必须在导入 goojia_utils 之前禁用 ES 8.x 的兼容性 Content-Type 头部请求! -# 这样客户端就会使用普通的 'application/json' 头部,彻底解决低版本 Elasticsearch 报错 406 的故障。 -os.environ["ELASTIC_CLIENT_APIVERSIONING"] = "0" -flag = False -for province in areautils.get_all_province(): - print(f"开始处理{province}数据") - # 百度经纬度转高德经纬度 - df = DIDATA_DEV_ELASTIC_CLIENT.scan_to_data_frame(index='didata_enterprise', query={ - "query": { - "bool": { - "must": [ - { - "term": { - "province": { - "value": "" + province + "" +# 🛡️ 2. 终极绝招:利用 Elasticsearch 8.x 官方 options() 机制,强行生成一个无 compatible 请求头的底层客户端并覆盖,彻底降维打击 406 报错! +DIDATA_DEV_ELASTIC_CLIENT.elastic_client = DIDATA_DEV_ELASTIC_CLIENT.elastic_client.options( + headers={"content-type": "application/json", "accept": "application/json"} +) + + +# ================== 🛡️ 安全运行入口 ================== + +if __name__ == "__main__": + flag = False + for province in areautils.get_all_province(): + print(f"开始处理{province}数据") + # 百度经纬度转高德经纬度 + df = DIDATA_DEV_ELASTIC_CLIENT.scan_to_data_frame(index='didata_enterprise', query={ + "query": { + "bool": { + "must": [ + { + "term": { + "province": { + "value": "" + province + "" + } + } } - } + ], + "must_not": [ + { + "exists": { + "field": "gaode_location" + } + } + ] } - ], - "must_not": [ - { - "exists": { - "field": "gaode_location" - } - } - ] - } - } - }) - if len(df) == 0: - continue - def deal_location(item): - if pd.isna(item['location']): + } + }) + if len(df) == 0: + continue + + def deal_location(item): + if pd.isna(item['location']): + return item + gd_location = geohashutils.bd_to_gd(item['location']['lon'], item['location']['lat']) + item['gaode_location'] = {'lon': gd_location[0], 'lat': gd_location[1]} return item - gd_location = geohashutils.bd_to_gd(item['location']['lon'], item['location']['lat']) - item['gaode_location'] = {'lon': gd_location[0], 'lat': gd_location[1]} - return item - my_df = threads.process_dataframe_multithread(df, deal_location, num_threads=100) - actions = list() - count = 0 - for index, item in my_df[my_df['gaode_location'].notna()].iterrows(): - doc_info = {'gaode_location': item['gaode_location']} + my_df = threads.process_dataframe_multithread(df, deal_location, num_threads=100) - action = { - "_op_type": "update", - "_index": "didata_enterprise", - "_id": item['_id'], - "doc": doc_info, - "_retry_on_conflict": 5 - } + actions = list() + count = 0 + for index, item in my_df[my_df['gaode_location'].notna()].iterrows(): + doc_info = {'gaode_location': item['gaode_location']} - actions.append(action) - if len(actions) == 5000: - for success, info in helpers.parallel_bulk( - DIDATA_DEV_ELASTIC_CLIENT.elastic_client, - actions, - thread_count=5, # 并行线程数 - chunk_size=1000 # 批量大小 - ): - if not success: - print(f"索引失败: {info}") + action = { + "_op_type": "update", + "_index": "didata_enterprise", + "_id": item['_id'], + "doc": doc_info, + "_retry_on_conflict": 5 + } + + actions.append(action) + if len(actions) == 5000: + for success, info in helpers.parallel_bulk( + # 此时这里的底层 elastic_client 将自动套用强行重置后的 JSON 请求头 + DIDATA_DEV_ELASTIC_CLIENT.elastic_client, + actions, + thread_count=5, # 并行线程数 + chunk_size=1000 # 批量大小 + ): + if not success: + print(f"索引失败: {info}") + count += len(actions) + print("更新:" + str(count)) + actions.clear() + + if len(actions) > 0: + helpers.bulk(DIDATA_DEV_ELASTIC_CLIENT.elastic_client, actions) count += len(actions) print("更新:" + str(count)) - actions.clear() - if len(actions) > 0: - helpers.bulk(DIDATA_DEV_ELASTIC_CLIENT.elastic_client, actions) - count += len(actions) - print("更新:" + str(count)) - actions.clear() \ No newline at end of file + actions.clear() \ No newline at end of file