diff --git a/dags/db/etl.company_report b/dags/db/etl.company_report index d1b1a30..12cd725 100644 --- a/dags/db/etl.company_report +++ b/dags/db/etl.company_report @@ -21,5 +21,10 @@ LEFT JOIN ( ) cr ON c.unified_code=cr.unified_code WHERE cr.mail_address!='' and cr.mail_address is not null; +# 合并 +optimize table etl.dwd_company final; + # 清洗企业通信地址 -ALTER TABLE etl.dwd_company update mail_address1=replaceRegexpAll(mail_address, '^(.{2,}(省|自治区))?(.{2,}市)?(.{2,}(区|县))?', '') WHERE 1=1 \ No newline at end of file +ALTER TABLE etl.dwd_company update mail_address1=replaceRegexpAll(mail_address, '^(.{2,}(省|自治区))?(.{2,}市)?(.{2,}(区|县))?', '') WHERE 1=1 + +INSERT INTO etl.dwm_company(unified_code, province, city, region, name, address) SELECT unified_code, province_name, city_name, region_name, ent_name, reg_addr1 FROM etl.dwd_company WHERE city_name !='' and region_name != '' and reg_addr1 != ''; diff --git a/dags/jh/merge/merge.py b/dags/jh/merge/merge.py index 29cc155..fac7d78 100644 --- a/dags/jh/merge/merge.py +++ b/dags/jh/merge/merge.py @@ -114,8 +114,8 @@ try: result.append('FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_' + date + '_0001.dat文件不存在') if 'FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_' + date + '_0001.dat' not in sub_dirs: result.append('FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_' + date + '_0001.dat文件不存在') - if 'FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_' + date + '_0001.dat' not in sub_dirs: - result.append('FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_' + date + '_0001.dat文件不存在') + # if 'FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_' + date + '_0001.dat' not in sub_dirs: + # result.append('FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_' + date + '_0001.dat文件不存在') if date.endswith('28'): if 'FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_' + date + '_0001.dat' not in sub_dirs: