Hbase数据导入导出

Posted 2020-08-20 超大的皮卡丘

平时用于从生产环境hbase到导出数据到测试环境。

导入数据：

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.HConnection;
import org.apache.hadoop.hbase.client.HConnectionManager;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.util.Bytes;


@SuppressWarnings("deprecation")
public class HbaseImport {
    
    public static void main(String args[]) throws Exception{
        Configuration conf = HBaseConfiguration.create();
        HConnection connection = HConnectionManager.createConnection(conf);
        List<Map<String,String>> datas = getDatas("d:\\hbaseData\\datas.txt",connection);
        wirteHbase(connection,"EVENT_LOG_LBS",datas);
    }
    
    public static List<Map<String,String>> getDatas(String filePath,HConnection connection) throws IOException{
        List<Map<String,String>> datas = new ArrayList<Map<String,String>>();
        File file = new File(filePath);
        BufferedReader br = new BufferedReader(new FileReader(file));
        String tr = null;
        while(((tr = br.readLine()) != null)){
            String subData = tr.substring(1);
            Map<String,String> data = new HashMap<String,String>();
            String[] ss = subData.split("\\|");
            for(String s : ss){
                String[] tds = s.split("=");
                String v = "";
                if(tds.length == 2){
                    v = tds[1];
                }
                data.put(tds[0], v);
            }
            datas.add(data);
        }
        br.close();
        return datas;
    }
    
    public static void wirteHbase(HConnection connection,String tableName,List<Map<String,String>>datas) throws IOException{
        HTableInterface t = connection.getTable(tableName);
        for(Map<String,String> map : datas){
            Set<String> ks = map.keySet();
            Put put = new Put(Bytes.toBytes(map.get("rowkey")));
            for(String key : ks){
                put.add(Bytes.toBytes("f1"),Bytes.toBytes(key),Bytes.toBytes(map.get(key)));
            }
            t.put(put);
        }
    }
}

导出数据：

package hbase;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.List;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.HConnection;
import org.apache.hadoop.hbase.client.HConnectionManager;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.util.Bytes;


@SuppressWarnings("deprecation")
public class HbaseExport {
	
	public static Date getPassSevenDays(int day){
		Calendar calendar = Calendar.getInstance();
		int year = calendar.get(Calendar.YEAR);
		int dayOfYear = calendar.get(Calendar.DAY_OF_YEAR);
		int j = 0;
		for(int i = 0;i < day; i++){
			calendar.set(Calendar.DAY_OF_YEAR, dayOfYear - j);
			if(calendar.get(Calendar.YEAR) < year){
				//跨年了
				j = 1;
				//更新 标记年
				year = year + 1;
				//重置日历
				calendar.set(year, Calendar.DECEMBER,31);
				//重新获取dayOfYear
				dayOfYear = calendar.get(Calendar.DAY_OF_YEAR);
			}else{
				j = j + 1;
			}
		}
		return calendar.getTime();
	}
	
	public static Scan setScanCondition(Scan scan) throws IOException{
		Date newDay = new Date();
		Date otherDays = getPassSevenDays(7);
		
		scan.setTimeRange(otherDays.getTime(),newDay.getTime());
		scan.addColumn("f1".getBytes(), "LS_certifier_no".getBytes());
		scan.addColumn("f1".getBytes(), "LS_location".getBytes());
		scan.addColumn("f1".getBytes(), "LS_phone_no".getBytes());
		scan.addColumn("f1".getBytes(), "LS_longitude".getBytes());
		scan.addColumn("f1".getBytes(), "LS_latitude".getBytes());
		scan.addColumn("f1".getBytes(), "date".getBytes());
		scan.addColumn("f1".getBytes(), "time".getBytes());
		scan.addColumn("f1".getBytes(), "hourOfDay".getBytes());

		return scan;
	}
	
	
	public static void main(String args[]) throws IOException{
		Configuration conf = HBaseConfiguration.create();
		HConnection connection = HConnectionManager.createConnection(conf);
		String tableName = "EVENT_LOG_LBS_HIS";
		HTableInterface table = connection.getTable(tableName);
		
		Scan scan = new Scan();
		setScanCondition(scan);
		
		ResultScanner rs = table.getScanner(scan);
		for(Result r : rs){
			List<String> lines = new ArrayList<String>();
			StringBuilder sb = new StringBuilder();
			sb.append(" rowkey=" + Bytes.toString(r.getRow()));
			for(Cell cell : r.rawCells()){
				String name = Bytes.toString(CellUtil.cloneQualifier(cell));
				String value = Bytes.toString(CellUtil.cloneValue(cell));
				System.out.println(name + "=" + value);
				sb.append("|" + name + "=" + value);
			}
			lines.add(sb.toString());
			System.out.println("--------------------------");
			writeFile(lines,"/home/hdfs/datas");
		}
		
	}
	
	public static void writeFile(List<String> lines,String filePath) throws FileNotFoundException{
		File file = new File(filePath);
		PrintWriter pw = new PrintWriter(new FileOutputStream(file,true));
		for(String line : lines){
			pw.append(line);
			pw.append("\n");
		}
		pw.flush();
		pw.close();
	}
	
}