From fe592e20acdcbb7a2806f47b82910a1af2416db5 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Fri, 17 Mar 2023 11:18:37 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=B8=AD=E9=97=B4=E8=A1=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/.airflowignore | 3 ++- dags/business/business/community/comm.py | 13 ++++++++++--- dags/db/tmp.community_summary | 18 ++++++++++++++++++ 3 files changed, 30 insertions(+), 4 deletions(-) create mode 100644 dags/db/tmp.community_summary diff --git a/dags/.airflowignore b/dags/.airflowignore index cfe3491..fece69f 100644 --- a/dags/.airflowignore +++ b/dags/.airflowignore @@ -5,4 +5,5 @@ mt/supply/* business/business/* jh/merge/* jh/para/* -jh/price/* \ No newline at end of file +jh/price/* +db/* \ No newline at end of file diff --git a/dags/business/business/community/comm.py b/dags/business/business/community/comm.py index c98b037..a1748fe 100644 --- a/dags/business/business/community/comm.py +++ b/dags/business/business/community/comm.py @@ -8,7 +8,7 @@ import logging import pandas as pd -from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils +from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, ClickHouseClient from elasticsearch import helpers import os import sys @@ -16,6 +16,7 @@ import sys log = logging.getLogger(__name__) seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') es206_client = ElasticSearchClient(host='192.168.0.206') +tmp_ck = ClickHouseClient(database='tmp') index_name = "community_summary" m_date = dateutils.iso_format() @@ -88,8 +89,11 @@ def deal_comm(community_summary_df, seaweed_df): def sync(city=None): log.info(f"开始同步小区基本属性至206 {index_name}...") seaweed_query = {'status': 0, 'cat': 'district'} + ck_sql = "ALTER TABLE tmp.community_summary DELETE WHERE 1=1 " if pd.notna(city): seaweed_query['city'] = city + ck_sql += " AND city='" + city + "'" + tmp_ck.execute(ck_sql) seaweed_df = pd.DataFrame( list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1, @@ -99,6 +103,7 @@ def sync(city=None): .batch_size(100000))) sql = f"select id from {index_name} " + if pd.notna(city): sql += f" where city='{city}'" community_summary_df = es206_client.scan_by_sql_to_data_frame(sql) @@ -162,8 +167,10 @@ def sync(city=None): log.info("插入:" + str(count)) actions.clear() - # 写入csv文件,防止小区变动导致的后续过程更新报错 - seaweed_df.to_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8', index=False) + # 写入csv文件,防止小区变动导致的后续过程更新报错, + #seaweed_df.to_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8', index=False) + # 写入中间表clickhouse,解决多worker节点问题 + tmp_ck.to_sql_plus(seaweed_df, 'community_summary') deal_comm(community_summary_df, seaweed_df) log.info(f"完成同步小区基本属性至206 {index_name}") diff --git a/dags/db/tmp.community_summary b/dags/db/tmp.community_summary new file mode 100644 index 0000000..96176b1 --- /dev/null +++ b/dags/db/tmp.community_summary @@ -0,0 +1,18 @@ +CREATE TABLE tmp.community_summary +( + _id String COMMENT '小区ID', + city String COMMENT '城市', + region String COMMENT '区域', + name String COMMENT '名称', + lng Nullable(Float32) COMMENT '经度', + lat Nullable(Float32) COMMENT '纬度', + alias Array(String) COMMENT '别名', + estate_type2 String COMMENT '物业类型', + build_type String COMMENT '建筑类型', + household_count String COMMENT '户数', + block String COMMENT '板块', + estate_charge String COMMENT '物业费', + finish_building_date String COMMENT '建成年代', + m_date DateTime('Asia/Shanghai') default now() COMMENT '更新时间' +) ENGINE ReplacingMergeTree(m_date) +PARTITION BY city PRIMARY KEY (city, region, name) ORDER BY (city, region, name); \ No newline at end of file