新增百度转高德
This commit is contained in:
@@ -1,3 +1,53 @@
|
|||||||
"""
|
"""
|
||||||
第一步: 将百度经纬度转为高德经纬度(只需跑批一次即可) TODO
|
第一步: 将百度经纬度转为高德经纬度(只需跑批一次即可) TODO
|
||||||
"""
|
"""
|
||||||
|
from goojia_utils import DIDATA_DEV_ELASTIC_CLIENT, threads, geohashutils, areautils
|
||||||
|
import pandas as pd
|
||||||
|
from elasticsearch import helpers
|
||||||
|
import logging
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
flag = False
|
||||||
|
for province in areautils.get_all_province():
|
||||||
|
print(f"开始处理{province}数据")
|
||||||
|
# 百度经纬度转高德经纬度
|
||||||
|
df = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame(f"select location from didata_enterprise where province='{province}'")
|
||||||
|
def deal_location(item):
|
||||||
|
if pd.isna(item['location']):
|
||||||
|
return item
|
||||||
|
gd_location = geohashutils.bd_to_gd(item['location']['lon'], item['location']['lat'])
|
||||||
|
item['gaode_location'] = {'lon': gd_location[0], 'lat': gd_location[1]}
|
||||||
|
return item
|
||||||
|
my_df = threads.process_dataframe_multithread(df, deal_location, num_threads=100)
|
||||||
|
|
||||||
|
actions = list()
|
||||||
|
count = 0
|
||||||
|
for index, item in my_df[my_df['gaode_location'].notna()].iterrows():
|
||||||
|
doc_info = {'gaode_location': item['gaode_location']}
|
||||||
|
|
||||||
|
action = {
|
||||||
|
"_op_type": "update",
|
||||||
|
"_index": "didata_enterprise",
|
||||||
|
"_id": item['_id'],
|
||||||
|
"doc": doc_info,
|
||||||
|
"_retry_on_conflict": 5
|
||||||
|
}
|
||||||
|
|
||||||
|
actions.append(action)
|
||||||
|
if len(actions) == 5000:
|
||||||
|
for success, info in helpers.parallel_bulk(
|
||||||
|
DIDATA_DEV_ELASTIC_CLIENT.elastic_client,
|
||||||
|
actions,
|
||||||
|
thread_count=5, # 并行线程数
|
||||||
|
chunk_size=1000 # 批量大小
|
||||||
|
):
|
||||||
|
if not success:
|
||||||
|
print(f"索引失败: {info}")
|
||||||
|
count += len(actions)
|
||||||
|
print("更新:" + str(count))
|
||||||
|
actions.clear()
|
||||||
|
if len(actions) > 0:
|
||||||
|
helpers.bulk(DIDATA_DEV_ELASTIC_CLIENT.elastic_client, actions)
|
||||||
|
count += len(actions)
|
||||||
|
print("更新:" + str(count))
|
||||||
|
actions.clear()
|
||||||
Reference in New Issue
Block a user