HDFS学习笔记hdfs_shell & JavaAPI

Posted 2020-09-27 yfceshi

FileSystem shell指令

• 官方文档: HDFS Commands Reference

appendToFile
cat
checksum
chgrp
chmod
chown
copyFromLocal
copyToLocal
count
cp
createSnapshot
deleteSnapshot
df
du
dus
expunge
find
get
getfacl
getfattr
getmerge
help
ls
lsr
mkdir
moveFromLocal
moveToLocal
mv
put
renameSnapshot
rm
rmdir
rmr
setfacl
setfattr
setrep
stat
tail
test
text
touchz
truncate
usage

FileSystem java API

測试代码

package hdfs;

import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.net.URI;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;

public class HdfsOperator {
    static final String PATH = "hdfs://Master:9000/";
    static final String DIR = "/d1";
    static final String FILE = "/d1/hello";
    static final String loadFile = "F:/readme.txt";
    static final String downFile = "F:/readme_tmp.txt";
    private static void getList(FileSystem fileSystem, String DIR) throws Exception {
        final FileStatus[] listStatus = fileSystem.listStatus(new Path(DIR));
        for(FileStatus fileStatus : listStatus) {
            String isDir = fileStatus.isDirectory() ? "dir":"file";
            final String permission = fileStatus.getPermission().toString();
            final short replication = fileStatus.getReplication();
            final long len = fileStatus.getLen();
            final String path = fileStatus.getPath().toString();
            System.out.println(isDir+"\t"+permission+"\t"+replication+"\t"+len+"\t"+path);
        }
    }
    private static void getData(FileSystem fileSystem, String FILE) throws Exception {
        final FSDataInputStream in = fileSystem.open(new Path(FILE));
        final FileOutputStream out = new FileOutputStream(downFile);
        IOUtils.copyBytes(in, out, 4096, true);
        System.out.println("get hdfs: " +FILE+ " success! " + "saved as"+ downFile);
    }
    private static void putData(FileSystem fileSystem, String src, String dest ) throws Exception {
        final FSDataOutputStream out = fileSystem.create(new Path(dest));
        final FileInputStream in = new FileInputStream(src);
        IOUtils.copyBytes(in, out, 4096, true);
    }
    private static void remove(FileSystem fileSystem, String DIR) throws Exception {
        boolean flag = fileSystem.delete(new Path(DIR), true);
        System.out.println("del " + DIR + (flag ?
 "  Success":"  failure") );
    }
    private static void create(FileSystem fileSystem, String DIR) throws Exception {
        if( fileSystem.exists(new Path(DIR)) == true) {
            remove(fileSystem, DIR);
        }
        fileSystem.mkdirs(new Path(DIR));
    }
    private static  FileSystem getFileSystem(String PATH) throws Exception {
        return FileSystem.get(new URI(PATH), new Configuration());

    }
    public static void main(String[] args) throws Exception {
        FileSystem fileSystem = getFileSystem(PATH);
        create(fileSystem, DIR);

        putData(fileSystem, loadFile, FILE);

        getData(fileSystem, FILE);

        getList(fileSystem, PATH);

        remove(fileSystem, FILE);
    }

}