优化中间表
This commit is contained in:
+2
-1
@@ -5,4 +5,5 @@ mt/supply/*
|
||||
business/business/*
|
||||
jh/merge/*
|
||||
jh/para/*
|
||||
jh/price/*
|
||||
jh/price/*
|
||||
db/*
|
||||
@@ -8,7 +8,7 @@
|
||||
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, ClickHouseClient
|
||||
from elasticsearch import helpers
|
||||
import os
|
||||
import sys
|
||||
@@ -16,6 +16,7 @@ import sys
|
||||
log = logging.getLogger(__name__)
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
tmp_ck = ClickHouseClient(database='tmp')
|
||||
|
||||
index_name = "community_summary"
|
||||
m_date = dateutils.iso_format()
|
||||
@@ -88,8 +89,11 @@ def deal_comm(community_summary_df, seaweed_df):
|
||||
def sync(city=None):
|
||||
log.info(f"开始同步小区基本属性至206 {index_name}...")
|
||||
seaweed_query = {'status': 0, 'cat': 'district'}
|
||||
ck_sql = "ALTER TABLE tmp.community_summary DELETE WHERE 1=1 "
|
||||
if pd.notna(city):
|
||||
seaweed_query['city'] = city
|
||||
ck_sql += " AND city='" + city + "'"
|
||||
tmp_ck.execute(ck_sql)
|
||||
|
||||
seaweed_df = pd.DataFrame(
|
||||
list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1,
|
||||
@@ -99,6 +103,7 @@ def sync(city=None):
|
||||
.batch_size(100000)))
|
||||
|
||||
sql = f"select id from {index_name} "
|
||||
|
||||
if pd.notna(city):
|
||||
sql += f" where city='{city}'"
|
||||
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
|
||||
@@ -162,8 +167,10 @@ def sync(city=None):
|
||||
log.info("插入:" + str(count))
|
||||
actions.clear()
|
||||
|
||||
# 写入csv文件,防止小区变动导致的后续过程更新报错
|
||||
seaweed_df.to_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8', index=False)
|
||||
# 写入csv文件,防止小区变动导致的后续过程更新报错,
|
||||
#seaweed_df.to_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8', index=False)
|
||||
# 写入中间表clickhouse,解决多worker节点问题
|
||||
tmp_ck.to_sql_plus(seaweed_df, 'community_summary')
|
||||
|
||||
deal_comm(community_summary_df, seaweed_df)
|
||||
log.info(f"完成同步小区基本属性至206 {index_name}")
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
CREATE TABLE tmp.community_summary
|
||||
(
|
||||
_id String COMMENT '小区ID',
|
||||
city String COMMENT '城市',
|
||||
region String COMMENT '区域',
|
||||
name String COMMENT '名称',
|
||||
lng Nullable(Float32) COMMENT '经度',
|
||||
lat Nullable(Float32) COMMENT '纬度',
|
||||
alias Array(String) COMMENT '别名',
|
||||
estate_type2 String COMMENT '物业类型',
|
||||
build_type String COMMENT '建筑类型',
|
||||
household_count String COMMENT '户数',
|
||||
block String COMMENT '板块',
|
||||
estate_charge String COMMENT '物业费',
|
||||
finish_building_date String COMMENT '建成年代',
|
||||
m_date DateTime('Asia/Shanghai') default now() COMMENT '更新时间'
|
||||
) ENGINE ReplacingMergeTree(m_date)
|
||||
PARTITION BY city PRIMARY KEY (city, region, name) ORDER BY (city, region, name);
|
||||
Reference in New Issue
Block a user