优化中间表

This commit is contained in:
shenjian@fangjia.com
2023-03-17 11:18:37 +08:00
parent 3572add969
commit fe592e20ac
3 changed files with 30 additions and 4 deletions
+2 -1
View File
@@ -5,4 +5,5 @@ mt/supply/*
business/business/*
jh/merge/*
jh/para/*
jh/price/*
jh/price/*
db/*
+10 -3
View File
@@ -8,7 +8,7 @@
import logging
import pandas as pd
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, ClickHouseClient
from elasticsearch import helpers
import os
import sys
@@ -16,6 +16,7 @@ import sys
log = logging.getLogger(__name__)
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
es206_client = ElasticSearchClient(host='192.168.0.206')
tmp_ck = ClickHouseClient(database='tmp')
index_name = "community_summary"
m_date = dateutils.iso_format()
@@ -88,8 +89,11 @@ def deal_comm(community_summary_df, seaweed_df):
def sync(city=None):
log.info(f"开始同步小区基本属性至206 {index_name}...")
seaweed_query = {'status': 0, 'cat': 'district'}
ck_sql = "ALTER TABLE tmp.community_summary DELETE WHERE 1=1 "
if pd.notna(city):
seaweed_query['city'] = city
ck_sql += " AND city='" + city + "'"
tmp_ck.execute(ck_sql)
seaweed_df = pd.DataFrame(
list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1,
@@ -99,6 +103,7 @@ def sync(city=None):
.batch_size(100000)))
sql = f"select id from {index_name} "
if pd.notna(city):
sql += f" where city='{city}'"
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
@@ -162,8 +167,10 @@ def sync(city=None):
log.info("插入:" + str(count))
actions.clear()
# 写入csv文件,防止小区变动导致的后续过程更新报错
seaweed_df.to_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8', index=False)
# 写入csv文件,防止小区变动导致的后续过程更新报错,
#seaweed_df.to_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8', index=False)
# 写入中间表clickhouse,解决多worker节点问题
tmp_ck.to_sql_plus(seaweed_df, 'community_summary')
deal_comm(community_summary_df, seaweed_df)
log.info(f"完成同步小区基本属性至206 {index_name}")
+18
View File
@@ -0,0 +1,18 @@
CREATE TABLE tmp.community_summary
(
_id String COMMENT '小区ID',
city String COMMENT '城市',
region String COMMENT '区域',
name String COMMENT '名称',
lng Nullable(Float32) COMMENT '经度',
lat Nullable(Float32) COMMENT '纬度',
alias Array(String) COMMENT '别名',
estate_type2 String COMMENT '物业类型',
build_type String COMMENT '建筑类型',
household_count String COMMENT '户数',
block String COMMENT '板块',
estate_charge String COMMENT '物业费',
finish_building_date String COMMENT '建成年代',
m_date DateTime('Asia/Shanghai') default now() COMMENT '更新时间'
) ENGINE ReplacingMergeTree(m_date)
PARTITION BY city PRIMARY KEY (city, region, name) ORDER BY (city, region, name);