Files
airflow/dags/db/etl.company_report
T
Jian Shen 83665e7e81 优化
2023-06-08 15:03:10 +08:00

31 lines
1.7 KiB
Plaintext

CREATE TABLE etl.company_report (
id String,
company_id String,
unified_code String,
year String,
company String,
mail_address String,
m_date DateTime('Asia/Shanghai') default now() COMMENT '更新时间'
) ENGINE ReplacingMergeTree(m_date)
PARTITION BY substring(unified_code, 2, 2) PRIMARY KEY (id) ORDER BY (id);
# 导入数据
clickhouse-client --format_csv_delimiter="@" --input_format_with_names_use_header=0 --query="INSERT INTO etl.company_report FORMAT CSV" --host=192.168.0.216 --password=goojia123456 < part-r-00000
# 关联导入,可能DataGrip客户端超时,就在ClickHouse-Client命令行运行即可
INSERT INTO etl.dwd_company(district, ent_name, reg_addr, unified_code, authority, region_code, reg_addr1, province_code, city_code, province_name, city_name, region_name, mail_address)
SELECT district, ent_name, reg_addr, unified_code, authority, region_code, reg_addr1, province_code, city_code, province_name, city_name, region_name, cr.mail_address
FROM etl.dwd_company c
LEFT JOIN (
SELECT unified_code, argMax(mail_address, year) mail_address, argMax(year, year) new_year FROM etl.company_report GROUP BY unified_code
) cr ON c.unified_code=cr.unified_code
WHERE cr.mail_address!='' and cr.mail_address is not null;
# 合并
optimize table etl.dwd_company final;
# 清洗企业通信地址
ALTER TABLE etl.dwd_company update mail_address1=replaceRegexpAll(mail_address, '^(.{2,}(省|自治区))?(.{2,}市)?(.{2,}(区|县))?', '') WHERE 1=1
INSERT INTO etl.dwm_company(unified_code, province, city, region, name, address) SELECT unified_code, province_name, city_name, region_name, ent_name, reg_addr1 FROM etl.dwd_company WHERE city_name !='' and region_name != '' and reg_addr1 != '';