Hbase 表的Rowkey设计避免数据热点

Posted 2021-01-25 perfectdata

一、案例分析

常见避免数据热点问题的处理方式有：加盐、哈希、反转等方法结合预分区使用。

由于目前原数据第一字段为时间戳形式，第二字段为电话号码，直接存储容易引起热点问题，通过加随机列、组合时间戳、字段反转的方式来设计Rowkey，来实现既能高效查询又能避免热点问题。（由于案例数据量小未进行预分区）

二、代码部分

  1 package beifeng.hadoop.hbase;
  2 import java.io.IOException;
  3 import java.text.SimpleDateFormat;
  4 import java.util.Date;
  5 import org.apache.hadoop.conf.Configuration;
  6 import org.apache.hadoop.conf.Configured;
  7 import org.apache.hadoop.fs.Path;
  8 import org.apache.hadoop.hbase.HBaseConfiguration;
  9 import org.apache.hadoop.hbase.HColumnDescriptor;
 10 import org.apache.hadoop.hbase.HTableDescriptor;
 11 import org.apache.hadoop.hbase.MasterNotRunningException;
 12 import org.apache.hadoop.hbase.TableName;
 13 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
 14 import org.apache.hadoop.hbase.client.HBaseAdmin;
 15 import org.apache.hadoop.hbase.client.Mutation;
 16 import org.apache.hadoop.hbase.client.Put;
 17 import org.apache.hadoop.hbase.mapreduce.TableOutputFormat;
 18 import org.apache.hadoop.hbase.mapreduce.TableReducer;
 19 import org.apache.hadoop.io.LongWritable;
 20 import org.apache.hadoop.io.NullWritable;
 21 import org.apache.hadoop.mapreduce.Job;
 22 import org.apache.hadoop.mapreduce.Mapper;
 23 import org.apache.hadoop.mapreduce.Reducer;
 24 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 25 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
 26 import org.apache.hadoop.util.Tool;
 27 import org.apache.hadoop.util.ToolRunner;
 28 import org.apache.hadoop.io.Text;
 29 
 30 /**
 31  * 遵循rowkey的设计原则
 32  *  1.rowkey不能过长
 33  *  2.唯一性，加随机列  md5
 34  *  3.注意避免产生数据热点
 35  *  4.满足更多的查询场景
 36  * @author Administrator
 37  *
 38  */
 39 public class LoadData extends Configured implements Tool {
 40 
 41     /**
 42      * 综合考虑 使用时间和手机 做组合key，能更好的满足应用场景
 43      * @author Administrator
 44      *
 45      */
 46     public static class LoadDataMapper extends Mapper<LongWritable, Text, LongWritable, Text> {
 47         //专门处理时间戳 =》标准时间格式
 48         SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHsss");
 49         private Text mapOutputValue = new Text();
 50         @Override
 51         protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, LongWritable, Text>.Context context)
 52                 throws IOException, InterruptedException {
 53             String line = value.toString();
 54             String[] splited = line.split("	");
 55             
 56             //将切分的第一个字段转成标准时间
 57             String formatDate = sdf.format(new Date(Long.parseLong(splited[0].trim())));
 58             //将手机号码反转
 59             String phoneNumber = splited[1].toString();
 60                         String reversePhoneNumber = new StringBuffer(phoneNumber).reverse().toString();
 61             
 62             String rowKeyString = reversePhoneNumber +"|"+formatDate;
 63             //反转手机号+“|”+时间 +正行内容拼接
 64             mapOutputValue.set(rowKeyString+"	"+ line);
 65             context.write(key, mapOutputValue);
 66         }
 67     }
 68     
 69     public static class LoadDataReuducer extends TableReducer<LongWritable, Text, NullWritable>{
 70 
 71         //设置HBase的列簇
 72             private static final String COLUMN_FAMAILY = "info";            
 73         @Override
 74         protected void reduce(LongWritable key, Iterable<Text> values,
 75                 Reducer<LongWritable, Text, NullWritable, Mutation>.Context context)
 76                 throws IOException, InterruptedException {
 77             for (Text value:values) {
 78                 String[] splited = value.toString().split("	");
 79                 String rowKey = splited[0];
 80             //    System.err.println(rowKey);
 81                 Put put = new Put(rowKey.getBytes());
 82                 //put.addColumn(COLUMN_FAMAILY.getBytes(),"row".getBytes(),value.getBytes());
 83                 put.add(COLUMN_FAMAILY.getBytes(), "reportTime".getBytes(), splited[1].getBytes());
 84                 put.add(COLUMN_FAMAILY.getBytes(), "apmac".getBytes(), splited[3].getBytes());
 85                 put.add(COLUMN_FAMAILY.getBytes(), "acmac".getBytes(), splited[4].getBytes());
 86                 put.add(COLUMN_FAMAILY.getBytes(), "host".getBytes(), splited[5].getBytes());
 87                 put.add(COLUMN_FAMAILY.getBytes(), "siteType".getBytes(), splited[6].getBytes());
 88                 put.add(COLUMN_FAMAILY.getBytes(), "upPackNum".getBytes(), splited[7].getBytes());
 89                 put.add(COLUMN_FAMAILY.getBytes(), "downPackNum".getBytes(), splited[8].getBytes());
 90                 put.add(COLUMN_FAMAILY.getBytes(), "unPayLoad".getBytes(), splited[9].getBytes());
 91                 put.add(COLUMN_FAMAILY.getBytes(), "downPayLoad".getBytes(), splited[10].getBytes());
 92                 put.add(COLUMN_FAMAILY.getBytes(),"httpStatus".getBytes(),splited[11].getBytes());
 93                 context.write(NullWritable.get(), put);
 94                 
 95             }
 96         }
 97     }
 98 
 99     public static void createTable(String tableName) throws MasterNotRunningException, ZooKeeperConnectionException, IOException {
100         Configuration  conf = HBaseConfiguration.create();
101         conf.set("hbase.zookeeper.quorum", "beifeng01");
102         
103         HBaseAdmin admin = new HBaseAdmin(conf);
104         
105         TableName tName = TableName.valueOf(tableName);
106         
107         HTableDescriptor htd = new HTableDescriptor(tName);
108         HColumnDescriptor hcd = new HColumnDescriptor("info");
109         htd.addFamily(hcd);
110         
111         if(admin.tableExists(tName)) {
112             System.out.println(tableName+"is exist,trying to recrate the table");
113             admin.disableTable(tName);
114             admin.deleteTable(tName);
115         } 
116         admin.createTable(htd);
117         System.out.println("create new table"+ " " + tableName);
118         
119     }
120     
121     public int run(String[] args) throws Exception {
122 
123     Configuration conf = this.getConf();
124         conf.set("hbase.zookeeper.quorum", "beifeng01"); 
125         conf.set(TableOutputFormat.OUTPUT_TABLE, "phoneLog"); 
126         
127         createTable("phoneLog"); 
128         
129         Job job = Job.getInstance(conf, this.getClass().getSimpleName());  
130         job.setJarByClass(this.getClass());  
131         job.setNumReduceTasks(1); 
132         
133         // map class  
134         job.setMapperClass(LoadDataMapper.class);  
135         job.setMapOutputKeyClass(LongWritable.class);  
136         job.setMapOutputValueClass(Text.class);  
137    
138         // reduce class  
139         job.setReducerClass(LoadDataReuducer.class);  
140         job.setOutputFormatClass(TableOutputFormat.class); 
141         
142         Path inPath = new Path(args[0]);  
143         FileInputFormat.addInputPath(job, inPath);
144    
145        boolean isSucced = job.waitForCompletion(true);
146        
147        return isSucced ? 0 : 1; 
148     }
149 
150     public static void main(String[] args) throws Exception {
151         Configuration conf = HBaseConfiguration.create(); 
152         
153         //指定HDFS数据地址
154         args = new String[] {"hdfs://hbase/data/input/HTTP_20130313143750.data"};  
155            int status = ToolRunner.run(
156                    conf, 
157                    new LoadData(), 
158                    args);
159              
160     System.exit(status);
161     }
162 }

运行完程序scan后Rowkey效果同设计一致