diff --git a/dags/company/company/bd2gaode.py b/dags/company/company/bd2gaode.py index 1434b3e..6fcb923 100644 --- a/dags/company/company/bd2gaode.py +++ b/dags/company/company/bd2gaode.py @@ -1,3 +1,53 @@ """ 第一步: 将百度经纬度转为高德经纬度(只需跑批一次即可) TODO -""" \ No newline at end of file +""" +from goojia_utils import DIDATA_DEV_ELASTIC_CLIENT, threads, geohashutils, areautils +import pandas as pd +from elasticsearch import helpers +import logging +log = logging.getLogger(__name__) + +flag = False +for province in areautils.get_all_province(): + print(f"开始处理{province}数据") + # 百度经纬度转高德经纬度 + df = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame(f"select location from didata_enterprise where province='{province}'") + def deal_location(item): + if pd.isna(item['location']): + return item + gd_location = geohashutils.bd_to_gd(item['location']['lon'], item['location']['lat']) + item['gaode_location'] = {'lon': gd_location[0], 'lat': gd_location[1]} + return item + my_df = threads.process_dataframe_multithread(df, deal_location, num_threads=100) + + actions = list() + count = 0 + for index, item in my_df[my_df['gaode_location'].notna()].iterrows(): + doc_info = {'gaode_location': item['gaode_location']} + + action = { + "_op_type": "update", + "_index": "didata_enterprise", + "_id": item['_id'], + "doc": doc_info, + "_retry_on_conflict": 5 + } + + actions.append(action) + if len(actions) == 5000: + for success, info in helpers.parallel_bulk( + DIDATA_DEV_ELASTIC_CLIENT.elastic_client, + actions, + thread_count=5, # 并行线程数 + chunk_size=1000 # 批量大小 + ): + if not success: + print(f"索引失败: {info}") + count += len(actions) + print("更新:" + str(count)) + actions.clear() + if len(actions) > 0: + helpers.bulk(DIDATA_DEV_ELASTIC_CLIENT.elastic_client, actions) + count += len(actions) + print("更新:" + str(count)) + actions.clear() \ No newline at end of file