From 0f98e1e93552c5309ce02dbb072187055db40fe3 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Wed, 12 Oct 2022 17:07:33 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=B0=8F=E5=8C=BA=E5=9F=BA?= =?UTF-8?q?=E6=9C=AC=E5=B1=9E=E6=80=A7=E5=90=8C=E6=AD=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/business/business/community/comm.py | 112 +++++++++--------- .../{ => community}/community_summary.py | 2 + dags/business/community_summary_dag.py | 2 +- 3 files changed, 59 insertions(+), 57 deletions(-) rename dags/business/business/{ => community}/community_summary.py (86%) diff --git a/dags/business/business/community/comm.py b/dags/business/business/community/comm.py index 79e3d56..8fb2019 100644 --- a/dags/business/business/community/comm.py +++ b/dags/business/business/community/comm.py @@ -6,38 +6,15 @@ import logging import pandas as pd from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils from elasticsearch import helpers -import datetime log = logging.getLogger(__name__) seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') es206_client = ElasticSearchClient(host='192.168.0.206') index_name = "community_summary" - -log.info(f"开始同步小区基本属性至206 {index_name}...") -seaweed_df = pd.DataFrame( - list(seaweed_conn.find({'cat': 'district', 'status': 0}, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1, - 'estate_type2': 1, 'build_type': 1, 'household_count': 1, - 'estate_charge': 1, 'finish_building_date': 1}) - .batch_size(100000))) - - -community_summary_df = es206_client.scan_by_sql_to_data_frame(f"select id from {index_name}") -seaweed_df['_id'] = seaweed_df['_id'].astype(str) -seaweed_df = seaweed_df.rename(columns={"_id": "id"}) - -if len(community_summary_df) > 0: - community_summary_df['flag'] = 1 - - seaweed_new_df = pd.merge(seaweed_df, community_summary_df, how='left', on=['id']) - seaweed_existed_df = seaweed_new_df[seaweed_new_df['flag'] == 1] - seaweed_not_existed_df = seaweed_new_df[seaweed_new_df['flag'] != 1] -else: - seaweed_existed_df = pd.DataFrame() - seaweed_not_existed_df = seaweed_df - m_date = dateutils.iso_format() + def build_doc(item): """ 构建字典 @@ -75,39 +52,62 @@ def build_doc(item): return doc_dict -actions = list() -# 对于存在的更新 -for index, item in seaweed_existed_df.iterrows(): - doc_info = build_doc(item) +def sync(): + log.info(f"开始同步小区基本属性至206 {index_name}...") + seaweed_df = pd.DataFrame( + list(seaweed_conn.find({'cat': 'district', 'status': 0}, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1, + 'estate_type2': 1, 'build_type': 1, + 'household_count': 1, + 'estate_charge': 1, 'finish_building_date': 1}) + .batch_size(100000))) - action = { - "_op_type": "update", - "_index": index_name, - "_id": item['id'], - "doc": doc_info - } + community_summary_df = es206_client.scan_by_sql_to_data_frame(f"select id from {index_name}") + seaweed_df['_id'] = seaweed_df['_id'].astype(str) + seaweed_df = seaweed_df.rename(columns={"_id": "id"}) - actions.append(action) - if len(actions) == 1000: + if len(community_summary_df) > 0: + community_summary_df['flag'] = 1 + + seaweed_new_df = pd.merge(seaweed_df, community_summary_df, how='left', on=['id']) + seaweed_existed_df = seaweed_new_df[seaweed_new_df['flag'] == 1] + seaweed_not_existed_df = seaweed_new_df[seaweed_new_df['flag'] != 1] + else: + seaweed_existed_df = pd.DataFrame() + seaweed_not_existed_df = seaweed_df + + actions = list() + # 对于存在的更新 + for index, item in seaweed_existed_df.iterrows(): + doc_info = build_doc(item) + + action = { + "_op_type": "update", + "_index": index_name, + "_id": item['id'], + "doc": doc_info + } + + actions.append(action) + if len(actions) == 1000: + helpers.bulk(es206_client.elastic_client, actions) + actions.clear() + + # 对于不存在的插入 + for index, item in seaweed_not_existed_df.iterrows(): + doc_info = build_doc(item) + + action = { + "_op_type": "index", + "_index": index_name, + "_id": item['id'], + "_source": doc_info + } + + actions.append(action) + if len(actions) == 1000: + helpers.bulk(es206_client.elastic_client, actions) + actions.clear() + if len(actions) > 0: helpers.bulk(es206_client.elastic_client, actions) actions.clear() - -# 对于不存在的插入 -for index, item in seaweed_not_existed_df.iterrows(): - doc_info = build_doc(item) - - action = { - "_op_type": "index", - "_index": index_name, - "_id": item['id'], - "_source": doc_info - } - - actions.append(action) - if len(actions) == 1000: - helpers.bulk(es206_client.elastic_client, actions) - actions.clear() -if len(actions) > 0: - helpers.bulk(es206_client.elastic_client, actions) - actions.clear() -log.info(f"完成同步小区基本属性至206 {index_name}") + log.info(f"完成同步小区基本属性至206 {index_name}") diff --git a/dags/business/business/community_summary.py b/dags/business/business/community/community_summary.py similarity index 86% rename from dags/business/business/community_summary.py rename to dags/business/business/community/community_summary.py index 10751a9..553484e 100644 --- a/dags/business/business/community_summary.py +++ b/dags/business/business/community/community_summary.py @@ -1,5 +1,6 @@ import logging import sys +import comm log = logging.getLogger(__name__) @@ -8,4 +9,5 @@ log.info("同步类型:" + type) if type == 'all': + comm.sync() pass \ No newline at end of file diff --git a/dags/business/community_summary_dag.py b/dags/business/community_summary_dag.py index 354d5bd..c1ee918 100644 --- a/dags/business/community_summary_dag.py +++ b/dags/business/community_summary_dag.py @@ -12,6 +12,6 @@ with DAG( ) as dag: t1 = BashOperator( task_id='community_summary_dag', - bash_command='python /opt/airflow/dags/repo/dags/business/business/community_summary.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/community_summary.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', ) t1 \ No newline at end of file