hive基本操作

Posted 2022-01-08 csyusu

一、创建分区分桶表

//clustered by (pnl_id) into 40 buckets 分成40个桶，动态分区如果分区列值太多，造成文件数过多引起系统崩溃等问题，因此可以对该列分桶，会根据hash值分桶。
create table test(
pnl_id string,
event_timekey string
)
partitioned by (factory string)
clustered by (pnl_id) into 40 buckets
row format serde ‘org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe‘
with serdeproperties (
‘field.delim‘=‘\t‘,
‘escape.delim‘=‘\n‘,
‘serialization.null.format‘=‘NULL‘,
‘serialization.encoding‘=‘UTF-8‘
) 
stored as parquet;

//select 源表 分区列跟据位置匹配，并不是根据名称
insert into test partition(factory) select pnl_id,event_timekey,factory as fac from dwr_pnl_hist where shift_timekey=‘20190610 180000‘;

set hive.exec.dynamic.partition=true;//开启动态分区
set hive.exec.dynamic.partition.mode=nonstrict;//非严格的分区模式 开启之后才能实现insert动态分区
set hive.exec.max.dynamic.partitions.pernode=100;//每个mapper或reducer可以创建的最大动态分区个数
set hive.exec.max.dynamic.partitions=1000//一个动态分区语句可创建动态分区个数
set hive.exec.max.created.files=100000//全局可创建最多文件个数

set hive.enforce.bucketing=true;//开启强制分桶 

二、常用查询命令

//sort by 每个reduce内部排序 order by 数据通过一个reduce全局排序
select * from test sort by event_timekey desc limit 10;