Java API实现Hadoop文件系统增删改查

Posted 2020-10-26 潇雨危栏

Java API实现Hadoop文件系统增删改查

Hadoop文件系统可以通过shell命令hadoop fs -xx进行操作，同时也提供了Java编程接口

maven配置

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>

  <groupId>hadoopStudy</groupId>
  <artifactId>demo</artifactId>
  <version>1.0-SNAPSHOT</version>
  <packaging>jar</packaging>

  <name>demo</name>
  <url>http://maven.apache.org</url>

  <properties>
    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
  </properties>

  <dependencies>
    <dependency>
      <groupId>junit</groupId>
      <artifactId>junit</artifactId>
      <version>3.8.1</version>
      <scope>test</scope>
    </dependency>

    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-common</artifactId>
      <version>3.0.0</version>
    </dependency>

    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-client</artifactId>
      <version>3.0.0</version>
    </dependency>

    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-hdfs</artifactId>
      <version>3.0.0</version>
    </dependency>
  </dependencies>
</project>

代码实现

package hadoopStudy;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

public class HdfsUtil {
    private FileSystem fs;
    HdfsUtil(String user){
        Configuration cfg = new Configuration();
        URI uri = null;
        try {
            uri = new URI("hdfs://localhost:9000");
        } catch (URISyntaxException e) {
            e.printStackTrace();
        }
        try {
            // 根据配置文件，实例化成DistributedFileSystem
            fs = FileSystem.get(uri, cfg, user); // 得到fs句柄
        } catch (IOException e) {
            e.printStackTrace();
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }

    /**
     * 上传文件
     */
    public void upload(String src, String dst){
        try {
            // 上传
            fs.copyFromLocalFile(new Path(src), new Path(dst));
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /**
     * 下载文件
     */
    public void download(String src, String dst){
        try {
            // 下载
            fs.copyToLocalFile(new Path(src), new Path(dst));
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /**
     * 创建文件夹
     */
    public void mkdir(String dir){
        try {
            fs.mkdirs(new Path(dir));
        } catch (IOException e) {
            // 创建目录
            e.printStackTrace();
        }
    }

    /**
     * 删除文件
     */
    public void delete(String name, boolean recursive){
        try {
            fs.delete(new Path(name), recursive);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /**
     * 重命名
     */
    public void rename(String source, String dst){
        try {
            fs.rename(new Path(source), new Path(dst));
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /**
     * 列出文件信息
     */
    public void list(String dir, boolean recursive){
        try {
            RemoteIterator<LocatedFileStatus> iter = fs.listFiles(new Path(dir), recursive);
            while (iter.hasNext()){
                LocatedFileStatus file = iter.next();
                System.out.println(file.getPath().getName());
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}

小结

Hdfs的Java API提供了优雅的FileSystem抽象类，在客户端使用时与其他文件系统区别不大