Files
airflow/dags/db/etl.company
T
shenjian@fangjia.com b915e1fe1b bak
2023-04-21 11:35:25 +08:00

68 lines
2.3 KiB
Plaintext

CREATE TABLE etl.company (
id Int64,
district String,
ent_name String,
reg_addr String,
unified_code String,
authority String
) ENGINE MySQL('192.168.0.20:3306', 'database', 'td_gov_company_basic', 'root', '12345678')
SETTINGS
connection_wait_timeout=9999999,
connect_timeout=9999999,
read_write_timeout=9999999,
connection_auto_close=false;
CREATE TABLE etl.dwd_company (
district String,
ent_name String,
reg_addr String,
unified_code String,
authority String,
m_date DateTime('Asia/Shanghai') default now() COMMENT '更新时间'
) ENGINE ReplacingMergeTree(m_date)
PARTITION BY district PRIMARY KEY (unified_code) ORDER BY (unified_code);
CREATE TABLE etl.dwm_company (
district String,
ent_name String,
reg_addr String,
unified_code String,
authority String,
m_date DateTime('Asia/Shanghai') default now() COMMENT '更新时间'
) ENGINE ReplacingMergeTree(m_date)
PARTITION BY district PRIMARY KEY (unified_code) ORDER BY (unified_code);
INSERT INTO etl.dwm_company(
district String,
ent_name String,
reg_addr String,
unified_code String,
authority String) SELECT * FROM etl.company
# 导入company.csv
clickhouse-client --format_csv_delimiter="|" --input_format_with_names_use_header=0 --query="INSERT INTO etl.dwd_company FORMAT CSVWithNames" --host=192.168.0.216 --password=goojia123456 < company.csv
# 导出company.csv
clickhouse-client --password goojia123456 -d etl -q "select * from company FORMAT CSVWithNames" --format_csv_delimiter='|' > company.csv
CREATE TABLE etl.company_local ON CLUSTER fangjia
(
district String,
ent_name String,
reg_addr String,
unified_code String,
authority String
) ENGINE=MergeTree PARTITION BY substring(unified_code, 2, 2) PRIMARY KEY(unified_code) ORDER BY (unified_code)
CREATE TABLE etl.company_shard ON CLUSTER fangjia
(
district String,
ent_name String,
reg_addr String,
unified_code String,
authority String
) ENGINE=Distributed(fangjia, etl, company, hiveHash(substring(unified_code, 2, 2)))
clickhouse-client --format_csv_delimiter="|" --input_format_with_names_use_header=0 --query="INSERT INTO etl.company_shard FORMAT CSVWithNames" --input_format_allow_errors_ratio=0.3 --host=192.168.0.216 --password=goojia123456 < td_gov_company_basic.csv