Hadoop实战-使用Eclipse开发Hadoop API程序

Posted 2020-09-13

一、准备运行所需Jar包

1）avro-1.7.4.jar

2）commons-cli-1.2.jar

3）commons-codec-1.4.jar

4）commons-collections-3.2.1.jar

5）commons-compress-1.4.1.jar

6）commons-configuration-1.6.jar

7）commons-io-2.4.jar

8）commons-lang-2.6.jar

9）commons-logging-1.2.jar

10）commons-math3-3.1.1.jar

11）commons-net-3.1.jar

12）curator-client-2.7.1.jar

13）curator-recipes-2.7.1.jar

14）gson-2.2.4.jar

15）guava-20.0.jar

16）hadoop-annotations-2.8.0.jar

17）hadoop-auth-2.8.0.jar

18）hadoop-common-2.8.0.jar

19）hadoop-hdfs-2.8.0.jar

20）hadoop-hdfs-client-2.8.0.jar

21）htrace-core4-4.0.1-incubating.jar

22）httpclient-4.5.2.jar

23）jackson-core-asl-1.9.13.jar

24）jackson-mapper-asl-1.9.13.jar

25）jersey-core-1.9.jar

26）jersey-json-1.9.jar

27）jersey-server-1.9.jar

28）jets3t-0.9.0.jar

29）jetty-6.1.26.jar

30）jetty-sslengine-6.1.26.jar

31）jetty-util-6.1.26.jar

32）jsch-0.1.51.jar

33）jsr305-3.0.0.jar

34）log4j-1.2.17.jar

35）protobuf-java-2.5.0.jar

36）servlet-api-2.5.jar

37）slf4j-api-1.7.21.jar

38）xmlenc-0.52.jar

 

二、复制集群文件到项目的src/main/resources目录下

core-site.xml

hdfs-site.xml

 

三、编写代码

public static void createFile(String dst, byte[] contents)
            throws IOException {
        String uri = "hdfs://master:9000/";
        Configuration config = new Configuration();
        FileSystem fs = FileSystem.get(URI.create(uri), config);

        // 列出hdfs上/user/fkong/目录下的所有文件和目录
        FileStatus[] statuses = fs.listStatus(new Path("/test/"));
        for (FileStatus status : statuses) {
            System.out.println("==================:" + status
                    + ":=================");
        }

        // 在hdfs的/user/fkong目录下创建一个文件，并写入一行文本
        FSDataOutputStream os = fs.create(new Path("/test/hadoop4.log"));
        os.write("my first hadoop file! 不错！".getBytes());
        os.flush();
        os.close();

        // 显示在hdfs的指定文件的内容
        InputStream is = fs.open(new Path("/test/hadoop4.log"));
        IOUtils.copyBytes(is, System.out, 1024, true);
    }

    public static void main(String[] args) throws IOException {
        
        createFile("/user/hadoop/test/", "hello world".getBytes());
    }