Hadoop之深入HDFS原理＜二＞

Posted 2022-03-16 月疯

HDFS命令行操作：

 

hadoop运维工程司需要掌握的命令：

 

HDFS JAVA API
API文档：
http://archive.cloudera.com/cdh5/cdh/5/hadoop-2.6.0-cdh5.9.0/api/index.html
访问入口类：FIleSystem 

创建目录：mkdirs
上传文件：create/put或copyFromLocalFile
列出目录内容：listStatus
显示文件或目录的元数据：getFileStatus
下载文件：open/get或copyToLocalFile
删除文件或目录：delete 

注意：Hadoop默认支持权限控制，可将其关闭
hadfs-site.xml文件;dfs.permissions.enabled设置成false 

文件系统的端口是50070

练习HDFSJava操作：maven依赖

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>groupId</groupId>
    <artifactId>hadoop</artifactId>
    <version>1.0-SNAPSHOT</version>
        <dependencies>
            <dependency>
                <groupId>org.apache.hadoop</groupId>
                <artifactId>hadoop-hdfs</artifactId>
                <version>2.9.0</version>
            </dependency>

            <dependency>
                <groupId>org.apache.hadoop</groupId>
                <artifactId>hadoop-client</artifactId>
                <version>2.9.0</version>
            </dependency>

            <dependency>
                <groupId>org.apache.hadoop</groupId>
                <artifactId>hadoop-common</artifactId>
                <version>2.9.0</version>
            </dependency>

            <dependency>
                <groupId>junit</groupId>
                <artifactId>junit</artifactId>
                <version>4.10</version>
                <scope>test</scope>
            </dependency>

            <dependency>
                <groupId>org.apache.hadoop</groupId>
                <artifactId>hadoop-mapreduce-client-core</artifactId>
                <version>2.9.0</version>
                <scope>provided</scope>
            </dependency>
            <dependency>
                <groupId>org.apache.hadoop</groupId>
                <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
                <version>2.9.0</version>
                <scope>provided</scope>
            </dependency>
        </dependencies>

</project>

 代码：

package com.hadoop;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;

public class Main 

    public static void main(String[] args) 
	// write your code here
    
    //生成文件系统
    public static FileSystem getHadoopFileSystem()
        FileSystem fs =null;
        //根据配置文件创建HDFS对象
        Configuration conf=new Configuration();
        conf.set("fs.defaultFS","hdfs://hadoop-senior.test.com:9000");
        //设置副本策略
        conf.set("dfs.replication","1");
        try
            //根据配置文件创建HDFS对象
            fs = FileSystem.get(conf);
        catch (Exception e)
            e.printStackTrace();
        
        return fs;
    
    //创建目录
    public static boolean createPath()
        boolean result = false;
        //获取文件系统
        FileSystem fs =getHadoopFileSystem();
        //调用系统文件系统mkdirs创建目录
        Path path=new Path("/test");
        try
            result = fs.mkdirs(path);
        catch (IOException e)
            e.printStackTrace();
        finally
            try
                //将文件关闭
                fs.close();
            catch (IOException e)
                e.printStackTrace();
            

        
        return result;
    
    //创建文件\\写入数据
    public static boolean createFile(String pathName)
        String content="nihao";
        boolean result = false;
        FileSystem fs =getHadoopFileSystem();
            try 
                FSDataOutputStream out=fs.create(new Path(pathName));
                out.writeUTF(content);
                result = true;
             catch (IOException e) 
                e.printStackTrace();
            finally
                try 
                    fs.close();
                 catch (IOException e) 
                    e.printStackTrace();
                
            

        return result;
    
    //本地上传文件到hdfs
    public static void putFileHDFS()
        FileSystem fs=getHadoopFileSystem();
        //如果上传的路径不存在会创建；如果该路径文件已存在，就会覆盖
        Path localPath=new Path("/usr/local/software/datas/test.txt");
        Path hdfsPath=new Path("/test/test.txt");
        try 
            //复制本地文件到HDFS
            fs.copyFromLocalFile(localPath,hdfsPath);
         catch (IOException e) 
            e.printStackTrace();
        finally 
            try 
                fs.close();
             catch (IOException e) 
                e.printStackTrace();
            
        
    
    //上传文件（通过输入和输出流）
    public static void putFileHDFS2(String outFile,String inFile)
        //获取文件系统
        FileSystem fileSystem=getHadoopFileSystem();
        //创建输出文件
        try
       //通过上传文件，生成输出流
            FSDataOutputStream out=fileSystem.create(new Path(outFile));
            //通过本地文件生成输入流
            FileInputStream in=new FileInputStream(inFile);
            //通过IOUtils的copyBytes方法传递数据流
            //buffSize:缓冲区大小
            //是否进行关闭，设置true，传输完毕，自动关闭输出输入流
            IOUtils.copyBytes(in,out,4096,true);
        catch(Exception e)
            e.printStackTrace();
        

    
    //获取上传文件的基本信息
    public static void list()
        FileSystem fs=getHadoopFileSystem();
        Path path=new Path("/test");
        try 
            //遍历目录，使用FileSystem的listStatus
            FileStatus[] fileStatuList = fs.listStatus(path);
            //使用FileStatus对象查看file状态
            for(FileStatus filestatus:fileStatuList)
                String isDir =filestatus.isDirectory()? "目录":"文件";
                String name = filestatus.getPath().toString();
                System.out.print(isDir+" "+name);
            
         catch (IOException e) 
            e.printStackTrace();
        finally
            try 
                fs.close();
             catch (IOException e) 
                e.printStackTrace();
            
        
    
    //下载文件
    public static void getFileFromHDFS()
        FileSystem fs =getHadoopFileSystem();
        Path HDFSPath=new Path("/test/test.txt");
        Path localPath=new Path("/usr/local/software/datas/test.txt");
        try 
            //false:不删除服务器文件
            //HDFSPath:本地目录
            //localPath：服务器目录
            //true:由于本地文件系统window系统，没有安装hadoop环境，所以使用第四个参数指定用本地文件系统
            fs.copyToLocalFile(false,HDFSPath,localPath,true);
         catch (IOException e) 
            e.printStackTrace();
        finally
            try 
                fs.close();
             catch (IOException e) 
                e.printStackTrace();
            
        
    
    //通过输入输出流下载文件
    public static void getFileFromHDFS1(String inFile,String outFile)
        FileSystem fs =getHadoopFileSystem();
        try 
            //hdfs文件通过输入流读取到内存
            FSDataInputStream in=fs.open(new Path(inFile));
            //内存中的数据通过输出流输出到本地
            FileOutputStream out=new FileOutputStream(outFile);
            IOUtils.copyBytes(in,out,4096,true);
            //也可以直接输出到控制台
//            IOUtils.copyBytes(in,System.out,4096,true);
         catch (IOException e) 
            e.printStackTrace();
        finally
            try 
                fs.close();
             catch (IOException e) 
                e.printStackTrace();
            
        
    
    //删除文件或者目录
    public static boolean  delete()
        boolean result =false;
        FileSystem fs =getHadoopFileSystem();
        Path path=new Path("/test/test.txt");
        try 
            result=fs.delete(path,true);
         catch (IOException e) 
            e.printStackTrace();
        finally
            try 
                fs.close();
             catch (IOException e) 
                e.printStackTrace();
            
        
        return result;