From c4368c14210eed3ad2d82d0fbe4180190db23747 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Thu, 5 Jun 2025 16:59:52 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=BE=A9sys=E6=8A=A5=E9=94=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/company/company/__init__.py | 0 dags/company/company/bd2gaode.py | 3 ++ dags/company/company/comm.py | 80 ++++++++++++++++++++++++++++++++ dags/company/dag_company.py | 18 +++++++ 4 files changed, 101 insertions(+) create mode 100644 dags/company/company/__init__.py create mode 100644 dags/company/company/bd2gaode.py create mode 100644 dags/company/company/comm.py create mode 100644 dags/company/dag_company.py diff --git a/dags/company/company/__init__.py b/dags/company/company/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dags/company/company/bd2gaode.py b/dags/company/company/bd2gaode.py new file mode 100644 index 0000000..dc59c9b --- /dev/null +++ b/dags/company/company/bd2gaode.py @@ -0,0 +1,3 @@ +""" +第一步: 将百度经纬度转为高德经纬度(只需跑批一次即可) +""" \ No newline at end of file diff --git a/dags/company/company/comm.py b/dags/company/company/comm.py new file mode 100644 index 0000000..ab34272 --- /dev/null +++ b/dags/company/company/comm.py @@ -0,0 +1,80 @@ +""" +通过居委围栏更新隶属关系 +""" +from goojia_utils import DIDATA_DEV_ELASTIC_CLIENT, areautils +import pandas as pd +from elasticsearch import helpers +import logging +log = logging.getLogger(__name__) + +def sync(province): + log.info(f"开始同步{province}企业数据的居委村委数据") + ids = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame(f"select company_name from didata_company where province='{province}'") + company_ids = ids['_id'].to_list() + + shape_df = DIDATA_DEV_ELASTIC_CLIENT.scan_by_sql_to_data_frame(f"select city_name, area_name, street_name, name, shape from community where province_name='{province}' ") + if len(shape_df) == 0: + log.info(f"{province}无围栏数据") + actions = list() + count = 0 + errors = list() + for index, item in shape_df[shape_df['shape'].notna()].iterrows(): + ret = DIDATA_DEV_ELASTIC_CLIENT.fetch_all_geo_points('didata_enterprise', item['shape']['coordinates'][0], field='gaode_location') + for info in ret: + source = info['_source'] + if 'district' in source and (source['district'] != item['area_name']): + errors.append(source) + continue + if ('registered_address' not in source) or ('company_name' not in source): + continue + doc_info = {'province': source['province'], 'city': source['city'], 'district': source['district'], + 'company_name': source['company_name'], 'registered_address': source['registered_address'], + 'location': source['gaode_location'], 'township': item['street_name'], + 'committee': item['name']} + if info['_id'] not in company_ids: + action = { + "_op_type": "index", + "_index": "didata_company", + "_id": info['_id'], + "_source": doc_info, + "_retry_on_conflict": 5 + } + else: + action = { + "_op_type": "update", + "_index": "didata_company", + "_id": info['_id'], + "doc": doc_info, + "_retry_on_conflict": 5 + } + + actions.append(action) + if len(actions) == 10000: + for success, info in helpers.parallel_bulk( + DIDATA_DEV_ELASTIC_CLIENT.elastic_client, + actions, + thread_count=5, # 并行线程数 + chunk_size=1000 # 批量大小 + ): + if not success: + print(f"索引失败: {info}") + count += len(actions) + print("更新:" + str(count)) + actions.clear() + #pd.DataFrame(errors)[['registered_address', 'company_name', 'province', 'city', 'district', 'gaode_location']].to_csv(f'{province}经纬度错误地址.csv', index=False) + if len(actions) > 0: + for success, info in helpers.parallel_bulk( + DIDATA_DEV_ELASTIC_CLIENT.elastic_client, + actions, + thread_count=5, # 并行线程数 + chunk_size=1000 # 批量大小 + ): + if not success: + print(f"索引失败: {info}") + count += len(actions) + print("更新:" + str(count)) + actions.clear() + #pd.DataFrame(errors)[['registered_address', 'company_name', 'province', 'city', 'district', 'gaode_location']].to_csv(f'{province}经纬度错误地址.csv', index=False) + log.info(f"完成同步{province}企业数据的居委村委数据") +for province in areautils.get_all_province(): + sync(province) diff --git a/dags/company/dag_company.py b/dags/company/dag_company.py new file mode 100644 index 0000000..c79e8dd --- /dev/null +++ b/dags/company/dag_company.py @@ -0,0 +1,18 @@ +from datetime import timedelta + +from airflow.operators.bash import BashOperator +import pendulum +from airflow import DAG + +with DAG( + dag_id="didata_company", + schedule_interval=None, + start_date=pendulum.datetime(2025, 6, 15, tz="Asia/Shanghai"), + catchup=True, + description="企业涉农处理" +) as dag: + t1 = BashOperator( + task_id='comm', + bash_command='python /opt/airflow/dags/repo/dags/company/company/comm.py', + ) + t1 \ No newline at end of file