版本修改
This commit is contained in:
@@ -1,80 +1,94 @@
|
||||
"""
|
||||
第一步: 将百度经纬度转为高德经纬度(只需跑批一次即可) TODO
|
||||
"""
|
||||
import os
|
||||
# 1. 尝试环境变量兼容
|
||||
os.environ["ELASTIC_CLIENT_APIVERSIONING"] = "0"
|
||||
|
||||
from goojia_utils import DIDATA_DEV_ELASTIC_CLIENT, threads, geohashutils, areautils
|
||||
import pandas as pd
|
||||
from elasticsearch import helpers
|
||||
import logging
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
import os
|
||||
# 🛡️ 极其重要:必须在导入 goojia_utils 之前禁用 ES 8.x 的兼容性 Content-Type 头部请求!
|
||||
# 这样客户端就会使用普通的 'application/json' 头部,彻底解决低版本 Elasticsearch 报错 406 的故障。
|
||||
os.environ["ELASTIC_CLIENT_APIVERSIONING"] = "0"
|
||||
|
||||
flag = False
|
||||
for province in areautils.get_all_province():
|
||||
print(f"开始处理{province}数据")
|
||||
# 百度经纬度转高德经纬度
|
||||
df = DIDATA_DEV_ELASTIC_CLIENT.scan_to_data_frame(index='didata_enterprise', query={
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{
|
||||
"term": {
|
||||
"province": {
|
||||
"value": "" + province + ""
|
||||
# 🛡️ 2. 终极绝招:利用 Elasticsearch 8.x 官方 options() 机制,强行生成一个无 compatible 请求头的底层客户端并覆盖,彻底降维打击 406 报错!
|
||||
DIDATA_DEV_ELASTIC_CLIENT.elastic_client = DIDATA_DEV_ELASTIC_CLIENT.elastic_client.options(
|
||||
headers={"content-type": "application/json", "accept": "application/json"}
|
||||
)
|
||||
|
||||
|
||||
# ================== 🛡️ 安全运行入口 ==================
|
||||
|
||||
if __name__ == "__main__":
|
||||
flag = False
|
||||
for province in areautils.get_all_province():
|
||||
print(f"开始处理{province}数据")
|
||||
# 百度经纬度转高德经纬度
|
||||
df = DIDATA_DEV_ELASTIC_CLIENT.scan_to_data_frame(index='didata_enterprise', query={
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{
|
||||
"term": {
|
||||
"province": {
|
||||
"value": "" + province + ""
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"must_not": [
|
||||
{
|
||||
"exists": {
|
||||
"field": "gaode_location"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"must_not": [
|
||||
{
|
||||
"exists": {
|
||||
"field": "gaode_location"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
})
|
||||
if len(df) == 0:
|
||||
continue
|
||||
def deal_location(item):
|
||||
if pd.isna(item['location']):
|
||||
}
|
||||
})
|
||||
if len(df) == 0:
|
||||
continue
|
||||
|
||||
def deal_location(item):
|
||||
if pd.isna(item['location']):
|
||||
return item
|
||||
gd_location = geohashutils.bd_to_gd(item['location']['lon'], item['location']['lat'])
|
||||
item['gaode_location'] = {'lon': gd_location[0], 'lat': gd_location[1]}
|
||||
return item
|
||||
gd_location = geohashutils.bd_to_gd(item['location']['lon'], item['location']['lat'])
|
||||
item['gaode_location'] = {'lon': gd_location[0], 'lat': gd_location[1]}
|
||||
return item
|
||||
my_df = threads.process_dataframe_multithread(df, deal_location, num_threads=100)
|
||||
|
||||
actions = list()
|
||||
count = 0
|
||||
for index, item in my_df[my_df['gaode_location'].notna()].iterrows():
|
||||
doc_info = {'gaode_location': item['gaode_location']}
|
||||
my_df = threads.process_dataframe_multithread(df, deal_location, num_threads=100)
|
||||
|
||||
action = {
|
||||
"_op_type": "update",
|
||||
"_index": "didata_enterprise",
|
||||
"_id": item['_id'],
|
||||
"doc": doc_info,
|
||||
"_retry_on_conflict": 5
|
||||
}
|
||||
actions = list()
|
||||
count = 0
|
||||
for index, item in my_df[my_df['gaode_location'].notna()].iterrows():
|
||||
doc_info = {'gaode_location': item['gaode_location']}
|
||||
|
||||
actions.append(action)
|
||||
if len(actions) == 5000:
|
||||
for success, info in helpers.parallel_bulk(
|
||||
DIDATA_DEV_ELASTIC_CLIENT.elastic_client,
|
||||
actions,
|
||||
thread_count=5, # 并行线程数
|
||||
chunk_size=1000 # 批量大小
|
||||
):
|
||||
if not success:
|
||||
print(f"索引失败: {info}")
|
||||
action = {
|
||||
"_op_type": "update",
|
||||
"_index": "didata_enterprise",
|
||||
"_id": item['_id'],
|
||||
"doc": doc_info,
|
||||
"_retry_on_conflict": 5
|
||||
}
|
||||
|
||||
actions.append(action)
|
||||
if len(actions) == 5000:
|
||||
for success, info in helpers.parallel_bulk(
|
||||
# 此时这里的底层 elastic_client 将自动套用强行重置后的 JSON 请求头
|
||||
DIDATA_DEV_ELASTIC_CLIENT.elastic_client,
|
||||
actions,
|
||||
thread_count=5, # 并行线程数
|
||||
chunk_size=1000 # 批量大小
|
||||
):
|
||||
if not success:
|
||||
print(f"索引失败: {info}")
|
||||
count += len(actions)
|
||||
print("更新:" + str(count))
|
||||
actions.clear()
|
||||
|
||||
if len(actions) > 0:
|
||||
helpers.bulk(DIDATA_DEV_ELASTIC_CLIENT.elastic_client, actions)
|
||||
count += len(actions)
|
||||
print("更新:" + str(count))
|
||||
actions.clear()
|
||||
if len(actions) > 0:
|
||||
helpers.bulk(DIDATA_DEV_ELASTIC_CLIENT.elastic_client, actions)
|
||||
count += len(actions)
|
||||
print("更新:" + str(count))
|
||||
actions.clear()
|
||||
actions.clear()
|
||||
Reference in New Issue
Block a user