31 lines
1.7 KiB
Plaintext
31 lines
1.7 KiB
Plaintext
CREATE TABLE etl.company_report (
|
|
id String,
|
|
company_id String,
|
|
unified_code String,
|
|
year String,
|
|
company String,
|
|
mail_address String,
|
|
m_date DateTime('Asia/Shanghai') default now() COMMENT '更新时间'
|
|
) ENGINE ReplacingMergeTree(m_date)
|
|
PARTITION BY substring(unified_code, 2, 2) PRIMARY KEY (id) ORDER BY (id);
|
|
|
|
# 导入数据
|
|
clickhouse-client --format_csv_delimiter="@" --input_format_with_names_use_header=0 --query="INSERT INTO etl.company_report FORMAT CSV" --host=192.168.0.216 --password=goojia123456 < part-r-00000
|
|
|
|
# 关联导入,可能DataGrip客户端超时,就在ClickHouse-Client命令行运行即可
|
|
INSERT INTO etl.dwd_company(district, ent_name, reg_addr, unified_code, authority, region_code, reg_addr1, province_code, city_code, province_name, city_name, region_name, mail_address)
|
|
SELECT district, ent_name, reg_addr, unified_code, authority, region_code, reg_addr1, province_code, city_code, province_name, city_name, region_name, cr.mail_address
|
|
FROM etl.dwd_company c
|
|
LEFT JOIN (
|
|
SELECT unified_code, argMax(mail_address, year) mail_address, argMax(year, year) new_year FROM etl.company_report GROUP BY unified_code
|
|
) cr ON c.unified_code=cr.unified_code
|
|
WHERE cr.mail_address!='' and cr.mail_address is not null;
|
|
|
|
# 合并
|
|
optimize table etl.dwd_company final;
|
|
|
|
# 清洗企业通信地址
|
|
ALTER TABLE etl.dwd_company update mail_address1=replaceRegexpAll(mail_address, '^(.{2,}(省|自治区))?(.{2,}市)?(.{2,}(区|县))?', '') WHERE 1=1
|
|
|
|
INSERT INTO etl.dwm_company(unified_code, province, city, region, name, address) SELECT unified_code, province_name, city_name, region_name, ent_name, reg_addr1 FROM etl.dwd_company WHERE city_name !='' and region_name != '' and reg_addr1 != '';
|