编程实现从hadoop上下载

Posted 2020-09-05 Kooing

下载

package hadoopTest;

import java.io.File;
import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

public class Merge {
    Path inputPath = new Path("/test4/text3");
    Path outputPath = new Path("e:/test1/text3");
    static int number = 0;

    public void doMerge() throws IOException {
        Configuration config = new Configuration();
        config.set("fs.default.name", "hdfs://192.168.20.128:9000");
        FileSystem fs = FileSystem.get(config);
        if (Merge.isExit(outputPath.toString())) {// 已经存在创建副本
            String tempPath = outputPath.toString();
            String[] paths = tempPath.split("\\\\/");
            number++;
            String newPath = paths[paths.length - 1] + "副本" + number;
            for (; Merge.isExit("e:/test1/" + newPath);) {
                number++;
                newPath = paths[paths.length - 1] + "副本" + number;
            }
            Path newOutputPath = new Path("e:/test1/" + newPath);
            fs.copyToLocalFile(false, inputPath, newOutputPath);
        } else {// 不存在
            fs.copyToLocalFile(false, inputPath, outputPath);
        }
    }

    private static boolean isExit(String path) {// 判断文件是否存在
        File file = new File(path);
        boolean exit = file.exists();
        return exit;
    }
}

View Code