diff --git a/dags/db/etl.company b/dags/db/etl.company index 651ad1a..85f6f2e 100644 --- a/dags/db/etl.company +++ b/dags/db/etl.company @@ -13,31 +13,40 @@ SETTINGS connection_auto_close=false; CREATE TABLE etl.dwd_company ( - district String, - ent_name String, - reg_addr String, - unified_code String, - authority String, + district String, + ent_name String, + reg_addr String, + unified_code String, + authority String, + region_code String, + reg_addr1 String, + province_code String, + city_code String, + province_name String, + city_name String, + region_name String, m_date DateTime('Asia/Shanghai') default now() COMMENT '更新时间' ) ENGINE ReplacingMergeTree(m_date) -PARTITION BY district PRIMARY KEY (unified_code) ORDER BY (unified_code); +PARTITION BY province_code PRIMARY KEY (unified_code) ORDER BY (unified_code); CREATE TABLE etl.dwm_company ( - district String, - ent_name String, - reg_addr String, unified_code String, - authority String, + province String, + city String, + region String, + name String, + address String, m_date DateTime('Asia/Shanghai') default now() COMMENT '更新时间' ) ENGINE ReplacingMergeTree(m_date) -PARTITION BY district PRIMARY KEY (unified_code) ORDER BY (unified_code); +PARTITION BY province PRIMARY KEY (unified_code) ORDER BY (unified_code); INSERT INTO etl.dwm_company( - district String, - ent_name String, - reg_addr String, - unified_code String, - authority String) SELECT * FROM etl.company + unified_code, + province, + city, + region, + name, + address) SELECT (unified_code, province_name, city_name, region_name, ent_name, reg_addr1) FROM etl.company WHERE city_name !='' and region_name != ''; # 导入company.csv clickhouse-client --format_csv_delimiter="|" --input_format_with_names_use_header=0 --query="INSERT INTO etl.dwd_company FORMAT CSVWithNames" --host=192.168.0.216 --password=goojia123456 < company.csv diff --git a/dags/db/etl.region b/dags/db/etl.region new file mode 100644 index 0000000..d3099a1 --- /dev/null +++ b/dags/db/etl.region @@ -0,0 +1,27 @@ +create table etl.region_local on cluster fangjia ( + province_code String, + fangjia_province String, + province_name String, + city_code String, + city_name String, + fangjia_city String, + region_code String, + fangjia_area String, + region_name String +) +ENGINE MergeTree() +PARTITION BY province_code PRIMARY KEY (region_code) ORDER BY (region_code); + + +CREATE TABLE etl.region_shard ON CLUSTER fangjia +( + province_code String, + fangjia_province String, + province_name String, + city_code String, + city_name String, + fangjia_city String, + region_code String, + fangjia_area String, + region_name String +) ENGINE=Distributed(fangjia, etl, region_local, hiveHash(province_code)) \ No newline at end of file diff --git a/dags/db/fangjia.township b/dags/db/fangjia.township deleted file mode 100644 index 804ba7f..0000000 --- a/dags/db/fangjia.township +++ /dev/null @@ -1,13 +0,0 @@ -create table fangjia.region ( - province_code String, - fangjia_province String, - province_name String, - city_code String, - city_name String, - fangjia_city String, - region_code String, - fangjia_area String, - region_name String -) -ENGINE MergeTree() -PARTITION BY province_code PRIMARY KEY (region_code) ORDER BY (region_code); \ No newline at end of file