From c6d34fe9cca37afed8238a6bc3c04ac9060f3c4f Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Wed, 12 Oct 2022 15:49:28 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=B0=8F=E5=8C=BA=E5=9F=BA?= =?UTF-8?q?=E6=9C=AC=E5=B1=9E=E6=80=A7=E5=90=8C=E6=AD=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/business/business/community/__init__.py | 0 dags/business/business/community/comm.py | 113 +++++++++++++++++++ dags/business/business/community_summary.py | 11 ++ dags/business/community_summary_dag.py | 17 +++ requirements.txt | 3 +- 5 files changed, 143 insertions(+), 1 deletion(-) create mode 100644 dags/business/business/community/__init__.py create mode 100644 dags/business/business/community/comm.py create mode 100644 dags/business/business/community_summary.py create mode 100644 dags/business/community_summary_dag.py diff --git a/dags/business/business/community/__init__.py b/dags/business/business/community/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dags/business/business/community/comm.py b/dags/business/business/community/comm.py new file mode 100644 index 0000000..79e3d56 --- /dev/null +++ b/dags/business/business/community/comm.py @@ -0,0 +1,113 @@ +""" +同步小区基本信息至ES206 +""" + +import logging +import pandas as pd +from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils +from elasticsearch import helpers +import datetime + +log = logging.getLogger(__name__) +seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') +es206_client = ElasticSearchClient(host='192.168.0.206') + +index_name = "community_summary" + +log.info(f"开始同步小区基本属性至206 {index_name}...") +seaweed_df = pd.DataFrame( + list(seaweed_conn.find({'cat': 'district', 'status': 0}, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1, + 'estate_type2': 1, 'build_type': 1, 'household_count': 1, + 'estate_charge': 1, 'finish_building_date': 1}) + .batch_size(100000))) + + +community_summary_df = es206_client.scan_by_sql_to_data_frame(f"select id from {index_name}") +seaweed_df['_id'] = seaweed_df['_id'].astype(str) +seaweed_df = seaweed_df.rename(columns={"_id": "id"}) + +if len(community_summary_df) > 0: + community_summary_df['flag'] = 1 + + seaweed_new_df = pd.merge(seaweed_df, community_summary_df, how='left', on=['id']) + seaweed_existed_df = seaweed_new_df[seaweed_new_df['flag'] == 1] + seaweed_not_existed_df = seaweed_new_df[seaweed_new_df['flag'] != 1] +else: + seaweed_existed_df = pd.DataFrame() + seaweed_not_existed_df = seaweed_df + +m_date = dateutils.iso_format() + +def build_doc(item): + """ + 构建字典 + :param item: + :return: + """ + doc_dict = dict() + doc_dict['id'] = item["id"] + doc_dict['m_date'] = m_date + doc_dict['city'] = item["city"] + doc_dict['region'] = item["region"] + doc_dict['name'] = item["name"] + doc_dict['estate_type'] = item['estate_type2'] + + build_type = commutils.format_build_type(item['build_type']) + if pd.notna(build_type): + doc_dict['build_type'] = build_type + + household_count = commutils.format_households(item['household_count']) + if pd.notna(household_count): + doc_dict['household_count'] = household_count + + estate_charge = commutils.format_property_fee(item['estate_charge']) + if pd.notna(estate_charge): + doc_dict['estate_charge'] = float(estate_charge) + + finish_building_date = commutils.format_finish_building_date(item['finish_building_date']) + if pd.notna(finish_building_date): + doc_dict["finish_building_date"] = int(finish_building_date) + + lng = commutils.format_lat_lng(item['lng']) + lat = commutils.format_lat_lng(item['lat']) + if pd.notna(lng) and pd.notna(lat): + doc_dict['location'] = {'lon': lng, 'lat': lat} + return doc_dict + + +actions = list() +# 对于存在的更新 +for index, item in seaweed_existed_df.iterrows(): + doc_info = build_doc(item) + + action = { + "_op_type": "update", + "_index": index_name, + "_id": item['id'], + "doc": doc_info + } + + actions.append(action) + if len(actions) == 1000: + helpers.bulk(es206_client.elastic_client, actions) + actions.clear() + +# 对于不存在的插入 +for index, item in seaweed_not_existed_df.iterrows(): + doc_info = build_doc(item) + + action = { + "_op_type": "index", + "_index": index_name, + "_id": item['id'], + "_source": doc_info + } + + actions.append(action) + if len(actions) == 1000: + helpers.bulk(es206_client.elastic_client, actions) + actions.clear() +if len(actions) > 0: + helpers.bulk(es206_client.elastic_client, actions) + actions.clear() +log.info(f"完成同步小区基本属性至206 {index_name}") diff --git a/dags/business/business/community_summary.py b/dags/business/business/community_summary.py new file mode 100644 index 0000000..10751a9 --- /dev/null +++ b/dags/business/business/community_summary.py @@ -0,0 +1,11 @@ +import logging +import sys + +log = logging.getLogger(__name__) + +type = sys.argv[1] if len(sys.argv) > 1 else 'all' +log.info("同步类型:" + type) + + +if type == 'all': + pass \ No newline at end of file diff --git a/dags/business/community_summary_dag.py b/dags/business/community_summary_dag.py new file mode 100644 index 0000000..354d5bd --- /dev/null +++ b/dags/business/community_summary_dag.py @@ -0,0 +1,17 @@ +from airflow.operators.bash import BashOperator +import pendulum +from airflow import DAG + +with DAG( + "community_summary_dag", + # 每月2日1点整分运行 + schedule_interval="0 1 2 * *", + start_date=pendulum.datetime(2022, 10, 12, tz="Asia/Shanghai"), + catchup=True, + description="地址输出信息分类数据定时更新" +) as dag: + t1 = BashOperator( + task_id='community_summary_dag', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community_summary.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + ) + t1 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 6ff14dd..cb9761c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ goojia_utils -nextcloud-api-wrapper \ No newline at end of file +nextcloud-api-wrapper +elasticsearch==7.11.0 \ No newline at end of file