Files
airflow/dags/company/company/bd2gaode.py
T
2026-07-22 16:13:44 +08:00

80 lines
2.6 KiB
Python

"""
第一步: 将百度经纬度转为高德经纬度(只需跑批一次即可) TODO
"""
from goojia_utils import DIDATA_DEV_ELASTIC_CLIENT, threads, geohashutils, areautils
import pandas as pd
from elasticsearch import helpers
import logging
log = logging.getLogger(__name__)
import os
# 🛡️ 极其重要:必须在导入 goojia_utils 之前禁用 ES 8.x 的兼容性 Content-Type 头部请求!
# 这样客户端就会使用普通的 'application/json' 头部,彻底解决低版本 Elasticsearch 报错 406 的故障。
os.environ["ELASTIC_CLIENT_APIVERSIONING"] = "0"
flag = False
for province in areautils.get_all_province():
print(f"开始处理{province}数据")
# 百度经纬度转高德经纬度
df = DIDATA_DEV_ELASTIC_CLIENT.scan_to_data_frame(index='didata_enterprise', query={
"query": {
"bool": {
"must": [
{
"term": {
"province": {
"value": "" + province + ""
}
}
}
],
"must_not": [
{
"exists": {
"field": "gaode_location"
}
}
]
}
}
})
if len(df) == 0:
continue
def deal_location(item):
if pd.isna(item['location']):
return item
gd_location = geohashutils.bd_to_gd(item['location']['lon'], item['location']['lat'])
item['gaode_location'] = {'lon': gd_location[0], 'lat': gd_location[1]}
return item
my_df = threads.process_dataframe_multithread(df, deal_location, num_threads=100)
actions = list()
count = 0
for index, item in my_df[my_df['gaode_location'].notna()].iterrows():
doc_info = {'gaode_location': item['gaode_location']}
action = {
"_op_type": "update",
"_index": "didata_enterprise",
"_id": item['_id'],
"doc": doc_info,
"_retry_on_conflict": 5
}
actions.append(action)
if len(actions) == 5000:
for success, info in helpers.parallel_bulk(
DIDATA_DEV_ELASTIC_CLIENT.elastic_client,
actions,
thread_count=5, # 并行线程数
chunk_size=1000 # 批量大小
):
if not success:
print(f"索引失败: {info}")
count += len(actions)
print("更新:" + str(count))
actions.clear()
if len(actions) > 0:
helpers.bulk(DIDATA_DEV_ELASTIC_CLIENT.elastic_client, actions)
count += len(actions)
print("更新:" + str(count))
actions.clear()