hive 存储格式及压缩

-- 设置参数
set hivevar:target_db_name=db_dw;
use ${hivevar:target_db_name};

-- 创建textfile表
create table file_format_textfile 
row format delimited fields terminated by '01'
stored as textfile
as
select * from smple_table;

-- 测试各种压缩的orc表
create table file_format_orc_zlib
row format delimited fields terminated by '01'
stored as orc tblproperties ("orc.compress"="ZLIB")
as
select * from file_format_textfile
;

create table file_format_orc_snappy
row format delimited fields terminated by '01'
stored as orc tblproperties ("orc.compress"="SNAPPY")
as
select * from file_format_orc_zlib
;

create table file_format_orc_none
row format delimited fields terminated by '01'
stored as orc tblproperties ("orc.compress"="NONE")
as
select * from file_format_orc_zlib
;

create table file_format_orc_default
row format delimited fields terminated by '01'
stored as orc
as
select * from file_format_orc_zlib
;

-- 测试各种压缩的parquet表
create table file_format_parquet_zlib
row format delimited fields terminated by '01'
stored as parquet tblproperties ("parquet.compress"="ZLIB")
as
select * from file_format_orc_zlib
;

create table file_format_parquet_snappy
row format delimited fields terminated by '01'
stored as parquet tblproperties ("parquet.compress"="SNAPPY")
as
select * from file_format_orc_zlib
;

create table file_format_parquet_none
row format delimited fields terminated by '01'
stored as parquet tblproperties ("parquet.compress"="NONE")
as
select * from file_format_orc_zlib
;

create table file_format_parquet_default
row format delimited fields terminated by '01'
stored as parquet
as
select * from file_format_orc_zlib
;

-- 测试各种压缩的rcfile表(可能参数没生效，各种压缩后大小一致)
create table file_format_rcfile_zlib
row format delimited fields terminated by '01'
stored as rcfile tblproperties ("rcfile.compress"="ZLIB")
as
select * from file_format_orc_zlib
;

create table file_format_rcfile_snappy
row format delimited fields terminated by '01'
stored as rcfile tblproperties ("rcfile.compress"="SNAPPY")
as
select * from file_format_orc_zlib
;

create table file_format_rcfile_none
row format delimited fields terminated by '01'
stored as rcfile tblproperties ("rcfile.compress"="NONE")
as
select * from file_format_orc_zlib
;

create table file_format_rcfile_default
row format delimited fields terminated by '01'
stored as rcfile
as
select * from file_format_orc_zlib
;

-- 查看各种压缩下的格式大小
dfs -du -s /user/hive/warehouse/${hivevar:target_db_name}.db/file_format_textfile;

dfs -du -s /user/hive/warehouse/${hivevar:target_db_name}.db/file_format_orc_zlib;
dfs -du -s /user/hive/warehouse/${hivevar:target_db_name}.db/file_format_orc_snappy;
dfs -du -s /user/hive/warehouse/${hivevar:target_db_name}.db/file_format_orc_none;
dfs -du -s /user/hive/warehouse/${hivevar:target_db_name}.db/file_format_orc_default;

dfs -du -s /user/hive/warehouse/${hivevar:target_db_name}.db/file_format_parquet_zlib;
dfs -du -s /user/hive/warehouse/${hivevar:target_db_name}.db/file_format_parquet_snappy;
dfs -du -s /user/hive/warehouse/${hivevar:target_db_name}.db/file_format_parquet_none;
dfs -du -s /user/hive/warehouse/${hivevar:target_db_name}.db/file_format_parquet_default;

dfs -du -s /user/hive/warehouse/${hivevar:target_db_name}.db/file_format_rcfile_zlib;
dfs -du -s /user/hive/warehouse/${hivevar:target_db_name}.db/file_format_rcfile_snappy;
dfs -du -s /user/hive/warehouse/${hivevar:target_db_name}.db/file_format_rcfile_none;
dfs -du -s /user/hive/warehouse/${hivevar:target_db_name}.db/file_format_rcfile_default;

-- 统计数据，原文件见文件中的附件