Java多线程断点下载文件并压缩

Posted 2020-12-11 21-gram

引言：使用多线程下载图片可以大幅度的提升下载速度，基于封装理念把可以抽离出来的代码全部抽离。

代码思路：就是先把需要下载的文件放入一个集合，再下载到一个临时目录，在压缩，最后删除临时目录中的下载文件。

下载

设置Header

import java.util.HashMap;

/**
 * @author XuYanK
 * @date 2020/5/18 9:08
 */
public class Header {
    public static final HashMap<String ,String> header = new HashMap<>();
    private Header() {
    }
    static {
        header.put("Accept-Encoding", "gzip, deflate");
        header.put("Accept-Language", "zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2");
        header.put("Cache-Control", "max-age=0");
        header.put("Connection", "keep-alive");
        header.put("User-Agent","Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:59.0) Gecko/20100101 Firefox/59.0");
        header.put("Accept",
                "image/gif,image/jpeg,image/pjpeg,image/pjpeg, "
                        + "application/x-shockwave-flash, application/xaml+xml, "
                        + "application/vnd.ms-xpsdocument, application/x-ms-xbap"
                        + "application/x-ms-application,application/vnd.ms-excel"
                        + "application/vnd.ms-powerpoint, application/msword,*/*");
    }
}

编写下载线程类 DowloadThred

import java.io.InputStream;
import java.io.RandomAccessFile;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.concurrent.CountDownLatch;

/**
 * @author XuYanK
 * @date 2020/5/18 9:10
 */
public class DownloadThread implements Runnable {

    //当前线程的下载位置
    private int startPos;
    //定义当前线程负责下载的文件大小
    private int currentPartSize;
    //当前线程需要下载的文件块,此类的实例支持对随机访问文件的读取和写入。
    private RandomAccessFile currentPart;
    //定义该线程已下载的字节数
    private int length;
    //线程计数器
    private CountDownLatch latch;

    private String path;

    public int getLength() {
        return length;
    }

    public DownloadThread(int startPos,int currentPartSize,RandomAccessFile currentPart, String path ,CountDownLatch latch){
        this.startPos = startPos;
        this.currentPartSize = currentPartSize;
        this.currentPart = currentPart;
        this.path = path;
        this.latch =latch;
    }
    @Override
    public void run(){
        InputStream inputStream = null;
        try{
            URL url = new URL(path);
            HttpURLConnection conn = (HttpURLConnection)url.openConnection();
            conn.setConnectTimeout(5 * 1000);
            //设置请求方法
            conn.setRequestMethod("GET");
            //设置请求属性
            Header.header.forEach((key, value) -> conn.setRequestProperty(key,value));
            inputStream = conn.getInputStream();
            //inputStream.skip(n);跳过和丢弃此输入流中数据的 n 个字节
            inputStream.skip(this.startPos);
            byte[] buffer = new byte[1024];
            int hasRead = 0;
            //读取网络数据写入本地
            while(length < currentPartSize && (hasRead = inputStream.read(buffer)) != -1){
                currentPart.write(buffer, 0, hasRead);
                length += hasRead;
            }
        }
        catch(Exception e){
            e.printStackTrace();
        }finally {
            try {
                currentPart.close();
                inputStream.close();
            }catch (Exception e){
                e.printStackTrace();
            }
            latch.countDown();
        }
    }

}

编写下载工具类 DownUtil

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.RandomAccessFile;
import java.math.BigDecimal;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.concurrent.CountDownLatch;


/**
 * @author XuYanK
 * @date 2020/5/18 9:10
 */
public class DownUtil {
    private Logger logger = LoggerFactory.getLogger(DownUtil.class);
    //定义下载路径
    private String path;
    //指定所下载的文件的保存位置
    private String targetFile;
    //定义下载线程的数量
    private int threadNum;
    //定义下载线程的对象
    private DownloadThread[] threads;
    //下载文件的总大小
    private int fileSize;
    //线程计数器
    private CountDownLatch latch;

    public DownUtil(String path, String targetFile, int threadNum){
        this.path = path;
        this.targetFile = targetFile;
        this.threadNum = threadNum;
        threads = new DownloadThread[threadNum];
        latch = new CountDownLatch(threadNum);
    }

    /**
     * 多线程下载文件
     * @throws Exception
     */
    public void downLoad() throws Exception{
        long t1 = System.currentTimeMillis();
        URL url = new URL(path);
        HttpURLConnection conn = (HttpURLConnection)url.openConnection();
        conn.setConnectTimeout(5 * 1000);
        //设置请求方法
//        conn.setRequestMethod("POST");
        //设置请求属性
        Header.header.forEach((key, value) -> conn.setRequestProperty(key,value));
        //得到文件大小
        fileSize = conn.getContentLength();
        conn.disconnect();
        int currentPartSize = fileSize / threadNum + 1;
        RandomAccessFile file = new RandomAccessFile(targetFile,"rw");
        //设置本地文件大小
        file.setLength(fileSize);
        file.close();
        for(int i = 0;i < threadNum;i++){
            //计算每个线程的下载位置
            int startPos = i * currentPartSize;
            //每个线程使用一个RandomAccessFile进行下载
            RandomAccessFile currentPart = new RandomAccessFile(targetFile,"rw");
            //定位该线程的下载位置
            currentPart.seek(startPos);
            //创建下载线程
            threads[i] = new DownloadThread(startPos, currentPartSize, currentPart , path , latch);
            Thread t = new Thread(threads[i]);
            t.start();
        }
        BigDecimal n = BigDecimal.ZERO;
        while (true){
            BigDecimal length = BigDecimal.ZERO;
            if (latch.getCount()==0 && BigDecimal.ONE.compareTo(n)<=0){break;}
            for (DownloadThread downloadThread:threads) {
                length = length.add(new BigDecimal(downloadThread.getLength()));
            }
            n = length.divide(new BigDecimal(fileSize), 4, BigDecimal.ROUND_HALF_UP);
            logger.info("已下载"+n.multiply(new BigDecimal("100"))+"%");
            Thread.sleep(1000);
        }
        long t2 = System.currentTimeMillis();

        logger.info("下载完成，文件大小为"+fileSize*1.0/(1024*1024)+"M，共用时"+(t2-t1)*1.0/1000+"秒");
        //线程等待,一个文件下载成功后继续下一个
        latch.await();
    }

    /**
     *  根据文件名获取文件类型
     * @param fileName
     * @return
     */
    public static String getFileType(String fileName) {
        return fileName.substring(fileName.lastIndexOf("."));
    }
}

 压缩

压缩工具类ZipUtils

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.*;
import java.util.ArrayList;
import java.util.List;
import java.util.zip.CRC32;
import java.util.zip.CheckedOutputStream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;

/**
 * @author XuYanK
 * @date 2020/5/18 9:45
 */
public class ZipUtils {
    private static Logger logger = LoggerFactory.getLogger(ZipUtils.class);
    /** 缓冲器大小 */
    private static final int BUFFER = 512;

    /**压缩得到的文件的后缀名*/
    private static final String SUFFIX=".zip";

    /**
     *文件写入压缩包
     * @param sb
     * @param zipname 压缩包名称
     * @throws IOException
     */
    public static void writeZip(StringBuffer sb, String zipname) throws IOException {
        String[] files = sb.toString().split(",");
        OutputStream os = new BufferedOutputStream(new FileOutputStream(zipname + SUFFIX));
        ZipOutputStream zos = new ZipOutputStream(os);
        byte[] buf = new byte[8192];
        int len;
        for (int i = 0; i < files.length; i++) {
            File file = new File(files[i]);
            if (!file.isFile()) {
                continue;
            }
            ZipEntry ze = new ZipEntry(file.getName());
            zos.putNextEntry(ze);
            BufferedInputStream bis = new BufferedInputStream(new FileInputStream(file));
            while ((len = bis.read(buf)) > 0) {
                zos.write(buf, 0, len);
            }
            zos.closeEntry();
        }
        zos.closeEntry();
        zos.close();
    }

    /**
     * 得到源文件路径的所有文件
     * @param dirFile 压缩源文件路径
     * */
    public static List<File>  getAllFile(File dirFile){

        List<File> fileList=new ArrayList<>();

        File[] files= dirFile.listFiles();
        //文件
        for(File file:files){
            if(file.isFile()){
                fileList.add(file);
                logger.info("文件添加到压缩列表:"+file.getName());
            }else {//目录
                //非空目录
                if(file.listFiles().length!=0){
                    //把递归文件加到fileList中
                    fileList.addAll(getAllFile(file));
                }else {//空目录
                    fileList.add(file);
                    logger.info("空目录添加到压缩列表:"+file.getName());
                }
            }
        }
        return fileList;
    }


    /**
     * 压缩文件
     * @param dirPath 压缩源文件路径
     * @param zipFileName 压缩目标文件路径
     * */
    public static void compress(String dirPath,String zipFileName) {

        //添加文件的后缀名
        zipFileName = zipFileName + SUFFIX;
        File dirFile = new File(dirPath);
        List<File> fileList = getAllFile(dirFile);

        byte[] buffer = new byte[BUFFER];
        ZipEntry zipEntry = null;
        //每次读取出来的长度
        int readLength = 0;

        try {
            // 对输出文件做CRC32校验
            CheckedOutputStream cos = new CheckedOutputStream(new FileOutputStream(
                    zipFileName), new CRC32());
            ZipOutputStream zos = new ZipOutputStream(cos);

            for (File file : fileList) {
                //若是文件，则压缩文件
                if (file.isFile()) {

                    zipEntry = new ZipEntry(getRelativePath(dirPath, file));
                    zipEntry.setSize(file.length());
                    zipEntry.setTime(file.lastModified());
                    zos.putNextEntry(zipEntry);

                    InputStream is = new BufferedInputStream(new FileInputStream(file));

                    while ((readLength = is.read(buffer, 0, BUFFER)) != -1) {
                        zos.write(buffer, 0, readLength);
                    }
                    is.close();
                    logger.info("file compress:" + file.getCanonicalPath());
                    System.out.println();
                } else {     //若是空目录，则写入zip条目中

                    zipEntry = new ZipEntry(getRelativePath(dirPath, file));
                    zos.putNextEntry(zipEntry);
                    logger.info("dir compress: " + file.getCanonicalPath() + "/");
                }
            }
            zos.close();  //最后得关闭流，不然压缩最后一个文件会出错
        } catch (FileNotFoundException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /**
     * 获取相对路径
     * @param dirPath 源文件路径
     * @param file 准备压缩的单个文件
     * */
    public static String getRelativePath(String dirPath,File file){
        File dirFile=new File(dirPath);
        String relativePath=file.getName();

        while (true){
            file=file.getParentFile();
            if(file==null) break;
            if(file.equals(dirFile)){
                break;
            }
            else {
                relativePath=file.getName()+"/"+relativePath;
            }
        }
        return relativePath;
    }

    /**
     * 删除文件夹目录下的文件
     * @param path
     * @return
     */
    public static boolean deleteDir(String path) {
        File file = new File(path);
        //判断是否待删除目录是否存在
        if (!file.exists()) {
            logger.warn("文件目录不存在");
            return false;
        }
        //取得当前目录下所有文件和文件夹
        String[] content = file.list();
        for (String name : content) {
            File temp = new File(path, name);
            //判断是否是目录
            if (temp.isDirectory()) {
                //递归调用，删除目录里的内容
                deleteDir(temp.getAbsolutePath());
                //删除空目录
                temp.delete();
            } else {
                //直接删除文件
                if (!temp.delete()) {
                    logger.warn("删不掉，使用强制删除方法GC()");
                    //强制删除
//                    System.gc();
//                    temp.delete();
                }
            }
        }
        return true;
    }


}

测试

main方法

import com.alibaba.fastjson.JSONObject;
import lombok.extern.slf4j.Slf4j;
import java.io.File;
import java.util.List;
import java.util.Map;

/**
 * @author XuYanK
 * @date 2020/5/18 9:13
 */
@Slf4j
public class MyDowloadTest {
    public static void main(String[] args) {

        //系统本地临时文件夹路径
        String targetFlie =System.getProperty("java.io.tmpdir")+File.separator+"testDown";
        //判断是否存在该文件夹
        File file = new File(targetFlie);
        if (!file.exists()) {
            file.mkdirs();
        }
        String listData = "[{
" +
                "		"userId": "1",
" +
                "		"username": "admin",
" +
                "		"password": "123456",
" +
                "		"imgpath": "http://c.hiphotos.baidu.com/zhidao/pic/item/54fbb2fb43166d22d552b941432309f79052d23a.jpg"
" +
                "	},
" +
                "	{
" +
                "		"userId": "2",
" +
                "		"username": "jacks",
" +
                "		"password": "111111",
" +
                "		"imgpath": "http://e.hiphotos.baidu.com/zhidao/pic/item/b8014a90f603738d8d7adbbbb31bb051f819ec73.jpg"
" +
                "	}
" +
                "]";
        //转换list对象
        List<Map> lists = JSONObject.parseArray(listData, Map.class);
        for (int i = 0; i < lists.size(); i++) {
            String imgpath = (String) lists.get(i).get("imgpath");
            //获取文件后缀
            String fileType = DownUtil.getFileType(imgpath);
            //多线程下载列表图片
            DownUtil downUtil = new DownUtil(
                    imgpath,
                    targetFlie + "\" + System.currentTimeMillis() + fileType,
                    8);
            try {
                downUtil.downLoad();
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
        String zipName = "测试";
        ZipUtils.compress(targetFlie, zipName);
        ZipUtils.deleteDir(targetFlie);
//        FileUtils.deleteQuietly(new File(targetFlie));

    }


}