HDFS的Java API（ Java API封装类）

Posted 2021-01-22 mtimeyu

HDFS的Java API

Java API封装类

package com.hadoop.utils;

import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;

import java.io.IOException;
import java.io.InputStream;
import java.net.URI;

public class HDFUtil {
    public static FileSystem getFlieSystem( String url) {
        //StringUtils中方法的操作对象是java.lang.String类型的对象，是JDK提供的String类型操作方法的补充
        if (StringUtils.isBlank(url)) {//判断某字符串是否为空或长度为0或由空白符(whitespace)构成
            return null;
        }
        Configuration conf=new Configuration();
        FileSystem fs =null;
        try {
                    URI  uri=new URI( url.trim());
                    fs=FileSystem.get(uri,conf);
        } catch (Exception e) {
            e.printStackTrace();
        }
        return  fs;
    }
    //获取文件系统
    public static FileSystem getFileSystem(String url, String user){
        if (StringUtils.isBlank(url)){
            return  null;
        }
        Configuration conf=new Configuration();
        FileSystem fs =null;
        try {
            URI  uri=new URI( url.trim());
            fs=FileSystem.get(uri,conf,user);
        } catch (Exception e) {
            e.printStackTrace();
        }

        return  fs;
    }
    /*
     * 创建目录
     */
     public static boolean mkdir(String path) throws IOException {
         FileSystem fs=getFileSystem(path,"root");
         boolean bl=fs.mkdirs(new Path(path));
         return bl;
     }
/**
 * 读文件
 *@param filePath
 * @throws IOException
 */
public static void readFile(String filePath) throws IOException{
    FileSystem fs = getFileSystem(filePath,"root");
    InputStream in=null;
    try{
        in=fs.open(new Path(filePath));
        IOUtils.copyBytes(in, System.out,4096,false);
    }catch(Exception e){
        System.out.println(e.getMessage());
    }finally{
        IOUtils.closeStream(in);
    }
}
    /**
     * 上传文件
     */
   public static void putFile(String localpath,String hdfsPath) throws IOException {
       FileSystem fs = getFileSystem(hdfsPath,"root");
       fs.copyFromLocalFile(new Path(localpath),new Path(hdfsPath));
       fs.close();
   }
/**
 * 下载文件
 *
 */
       public  static  void getFile(String hdsPath,String localPath) throws IOException {
             FileSystem fs=getFileSystem(hdsPath,"root");
             Path hdfs_path=new Path(hdsPath);
             Path local_path=new Path(localPath);
            fs.copyToLocalFile(false,hdfs_path,local_path,true);
             fs.close();
       }
    /**
     * 递归删除
     */
    public static boolean deleteFile(String hdfsPath) throws IOException {
        FileSystem fs=getFileSystem(hdfsPath,"root");
        return  fs.delete( new Path(hdfsPath),true);

    }
    /**
     * 目录列表
     */
     public static String[] ListFile(String hdfsPaht){
          String [] file =new String[0];
          FileSystem fs= getFileSystem(hdfsPaht,"root");
          Path path=new Path(hdfsPaht);
          FileStatus[] ft=null;
         try {
              ft=fs.listStatus(path);
         } catch (IOException e) {
             e.printStackTrace();
         }
         file =new String[ft.length];
         for (int i=0;i<ft.length;i++){
             file[i] =ft[i].toString();
         }
         return  file;
     }
    /**
     * 主方法，测试
     */
    public static void main(String[] args) throws IOException {
         String url="hdfs://192.168.55.128:9000/";
         HDFUtil.mkdir(url+"util");
        // HDFUtil.putFile("D:\\\\words",url+"util/");
       //  HDFUtil.readFile(url+"util/words/words.txt");
        // HDFUtil.getFile(url+"util/words","D:\\\\util");
        // HDFUtil.deleteFile(url+"abc");
    }
}

 执行main方法前

[root@node1 ~]# hdfs dfs -ls /
Found 3 items
drwxr-xr-x   - root supergroup          0  20:52 /abc
drwxr-xr-x   - root supergroup          0  07:27 /input
drwxr-xr-x   - root supergroup          0  10:01 /user

 执行main方法后

 

修改主方法

public static void main(String[] args) throws IOException {
         String url="hdfs://192.168.55.128:9000/";
        // HDFUtil.mkdir(url+"util");
        //HDFUtil.putFile("D:\\\\words",url+"util/");
       // HDFUtil.readFile(url+"util/words/words.txt");
       //  HDFUtil.getFile(url+"util/words","D:\\\\util");
        //HDFUtil.deleteFile(url+"abc");
        String [] array=HDFUtil.ListFile(url);
        for (String ar: array) {
            System.out.println(ar);
            
        }