Hadoop学习笔记HDFS Java API

Posted 2021-08-29 500年

Hadoop学习笔记（五）HDFS Java API

springboot项目操作hdfs

pom.xml

<project xmlns="http://maven.apache.org/POM/4.0.0"
	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
	<modelVersion>4.0.0</modelVersion>
	<parent>
		<groupId>com.asia.common</groupId>
		<artifactId>cm-java-tip</artifactId>
		<version>0.0.1-SNAPSHOT</version>
	</parent>
	<artifactId>cm-java-tip-hdfs</artifactId>
	
	<dependencies>
		<dependency>
			<groupId>org.apache.hadoop</groupId>
			<artifactId>hadoop-common</artifactId>
			<version>3.2.0</version>
		</dependency>
		
		<dependency>
			<groupId>org.apache.hadoop</groupId>
			<artifactId>hadoop-client</artifactId>
			<version>3.2.0</version>
		</dependency>
		<dependency>
			<groupId>org.apache.hadoop</groupId>
			<artifactId>hadoop-hdfs</artifactId>
			<version>3.2.0</version>
		</dependency>
		<dependency>
			<groupId>junit</groupId>
			<artifactId>junit</artifactId>
			<version>RELEASE</version>
		</dependency>
	</dependencies>
</project>

工具类

package com.asia.tip;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;

import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;

public class MkdirDemo {

	static Configuration conf = null;
	static FileSystem fs = null;

	static {
		conf = new Configuration();
		conf.set("fs.defaultFS", "hdfs://127.0.0.1:9000");
		try {
			fs = FileSystem.get(conf);
		} catch (IOException e) {
			e.printStackTrace();
		}
	}

	public static void list() {
		try {
			RemoteIterator<LocatedFileStatus> listFiles = fs.listFiles(new Path("/"), true);
			while (listFiles.hasNext()) {
				LocatedFileStatus file = listFiles.next();
				System.out.println(file.getPath().toString());
			}
			fs.close();
		} catch (IOException e) {
			e.printStackTrace();
		}
	}

	public static void mkdir() {
		try {
			fs.mkdirs(new Path("/javadir1"));
			fs.close();
			System.out.println("创建目录成功！");
		} catch (IOException e) {
			e.printStackTrace();
		}
	}

	public static void upload() {
		try {
			fs.copyFromLocalFile(new Path("D:/tmp/yu7.jpg"), new Path("/javadir3/"));
			fs.close();
			System.out.println("上传文件成功！");
		} catch (Exception e) {
			e.printStackTrace();
		}
	}

	// 最简单的下载方式
	public static void download() {
		try {
			fs.copyToLocalFile(new Path("/javadir3/yu7.jpg"), new Path("D:/tmp/yu7.jpg"));
			fs.close();
			System.out.println("下载文件成功！");
		} catch (Exception e) {
			e.printStackTrace();
		}
	}
	
	public static void download2() {
		try {
			FSDataInputStream inputStream = fs.open(new Path("/javadir3/yu7.jpg"));
			FileOutputStream outputStream = new FileOutputStream(new File("D:/tmp/www.jpg"));
			IOUtils.copy(inputStream, outputStream);
			IOUtils.closeQuietly(inputStream);
			IOUtils.closeQuietly(outputStream);
			fs.close();
			System.out.println("下载文件成功2！");
		} catch (Exception e) {
			e.printStackTrace();
		}
	}

	public static void rename() {
		try {
			fs.rename(new Path("/javadir3/yu7.jpg"), new Path("/javadir3/yu7-1.jpg"));
			fs.close();
			System.out.println("文件改名成功！");
		} catch (Exception e) {
			e.printStackTrace();
		}
	}

	public static void delete() {
		try {
			fs.delete(new Path("/javadir3/yu7-1.jpg"), true);
			fs.close();
			System.out.println("删除文件成功！");
		} catch (Exception e) {
			e.printStackTrace();
		}
	}

	public static void main(String[] args) {
		// mkdir();
		download2();
	}

}