From 02e1aab54229abb9dce8d39cd8e95140dbd367f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 6 Jun 2025 15:23:05 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E7=99=BE=E5=BA=A6=E8=BD=AC?= =?UTF-8?q?=E9=AB=98=E5=BE=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/company/company/bd2gaode.py | 52 +++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/dags/company/company/bd2gaode.py b/dags/company/company/bd2gaode.py index 1434b3e..6fcb923 100644 --- a/dags/company/company/bd2gaode.py +++ b/dags/company/company/bd2gaode.py @@ -1,3 +1,53 @@ """ 第一步: 将百度经纬度转为高德经纬度(只需跑批一次即可) TODO -""" \ No newline at end of file +""" +from goojia_utils import DIDATA_DEV_ELASTIC_CLIENT, threads, geohashutils, areautils +import pandas as pd +from elasticsearch import helpers +import logging +log = logging.getLogger(__name__) + +flag = False +for province in areautils.get_all_province(): + print(f"开始处理{province}数据") + # 百度经纬度转高德经纬度 + df = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame(f"select location from didata_enterprise where province='{province}'") + def deal_location(item): + if pd.isna(item['location']): + return item + gd_location = geohashutils.bd_to_gd(item['location']['lon'], item['location']['lat']) + item['gaode_location'] = {'lon': gd_location[0], 'lat': gd_location[1]} + return item + my_df = threads.process_dataframe_multithread(df, deal_location, num_threads=100) + + actions = list() + count = 0 + for index, item in my_df[my_df['gaode_location'].notna()].iterrows(): + doc_info = {'gaode_location': item['gaode_location']} + + action = { + "_op_type": "update", + "_index": "didata_enterprise", + "_id": item['_id'], + "doc": doc_info, + "_retry_on_conflict": 5 + } + + actions.append(action) + if len(actions) == 5000: + for success, info in helpers.parallel_bulk( + DIDATA_DEV_ELASTIC_CLIENT.elastic_client, + actions, + thread_count=5, # 并行线程数 + chunk_size=1000 # 批量大小 + ): + if not success: + print(f"索引失败: {info}") + count += len(actions) + print("更新:" + str(count)) + actions.clear() + if len(actions) > 0: + helpers.bulk(DIDATA_DEV_ELASTIC_CLIENT.elastic_client, actions) + count += len(actions) + print("更新:" + str(count)) + actions.clear() \ No newline at end of file