多线程下载图片并压缩
Posted 杨吃羊
tags:
篇首语：本文由小常识网(cha138.com)小编为大家整理，主要介绍了多线程下载图片并压缩相关的知识，希望对你有一定的参考价值。
从ftp服务器下载文件,文件里是多张图片的name和uri,格式是 name||uri,根据uri下载图片并压缩存储到本地服务器
主要流程: 1 从ftp服务器下载文件, 2 将文件格式解析后的地址存入map容器内, 3 计算每个线程需要下载的url,并循环调用threadPool.execute(mythead),子线程根据uri下载字节流到内存中.4 用ZipOutputStream流压缩,
代码:
package kun.threadSafe; import org.quartz.JobExecutionContext; import org.quartz.JobExecutionException; import org.springframework.scheduling.quartz.QuartzJobBean; import java.io.*; import java.net.URL; import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; public class PictureTransferJob extends QuartzJobBean protected void executeInternal(JobExecutionContext arg0) throws JobExecutionException //实际的FTP配置是读取配置文件获取的 //FTP地址 String hostName = "192.168.x.xxx"; //FTP端口 int port = xxxx; //FTP账号 String userName = "xxx"; //ftp密码 String password = "xxx"; //ftp文件存储目录 String ftpDowload = "/"; //文件本地存储路径 String path = this.getClass().getResource("/").getPath(); //图片地址文件存储目录 String addrPath = path.substring(1, path.indexOf("WEB-INF/classes")) + "picAddr"; //实际下载的图片存储目录 String picPath = path.substring(1, path.indexOf("WEB-INF/classes")) + "pic"; addrPath = addrPath.replace("%20", " "); picPath = picPath.replace("%20", " "); try //创建存储图片地址的文件 creatFile(addrPath); //创建存储实际图片的文件 creatFile(picPath); String oldAddrPath = addrPath; String oldPicPath = picPath; //创建FTP连接 FtpUtil2 ftpUtil2 = new FtpUtil2(hostName, port, userName, password, ftpDowload, true); //遍历FTP目录下的文件 String[] files = ftpUtil2.ListAllFiles(); //本地数据库会有一个表记录下载过的文件，这里会查询数据库和ftp列出的文件名比较，如果已经下载过的文件就不会下载，避免重复下载。 //下面省略比较的过程，循环files数组，在本地创建文件 for (int i = 0; i < files.length; i++) creatFile(addrPath + File.separator + fileName); //ftpDowload是ftp服务器存储文件的地址，addrPath是本地存储文件的地址 //这里一个返回状态判断文件是否下载成功 boolean downloadInvestorFlag = ftpUtil2.downloadFile(ftpDowload, addrPath); //文件下载成功后调读取文件的方法，将需要下载的图片地址存入容器 boolean entityState = setPictureDetail(addrPath, picPath, fileNameDate); catch (Exception e) e.printStackTrace(); //调记录错误日志的业务类用于发送下载文件出错的短信 //这里开始读图片地址 private boolean setPictureDetail(String addrPath, String picPath, String synDate) System.out.println("----------进入setPictureDetail方法-----------"); BufferedReader br = null; try br = new BufferedReader(new InputStreamReader(new FileInputStream(addrPath), "UTF-8")); String row; int count = 0; //map中存储每行读取到的图片名称和URL地址 Map<String, String> addrMap = new HashMap<String, String>(); while ((row = br.readLine()) != null) try count++; if (count == 1) continue; String[] column = row.split("\\\\|\\\\|", -1); addrMap.put(column[0].trim(), column[1].trim()); catch (Exception e) e.printStackTrace(); System.out.println(new Date()); //这里调用压缩方法，压缩方法中会调用执行下载图片的方法 zipPic(picPath, synDate, addrMap); System.out.println(new Date()); System.out.println("----------完成--------------"); return true; catch (Exception e) e.printStackTrace(); //调用记录错误日志的业务类 return false; finally try if (null != br) br.close(); catch (IOException e) e.printStackTrace(); //这里是压缩文件的伪代码 private boolean zipPic(String picPath, String synDate, Map addrMap) //传入需要压缩的文件列表和压缩文件名 //这里由于是多线程存储图片流，所以需要使用线程安全的map，因此使用ConcurrentHashMap Map<String,InputStream> pictureList=new ConcurrentHashMap<String,InputStream>(); //这里定义每个线程下载的图片个数 int count=400; //存储需要下载的图片地址 List<Map.Entry<String, String>> addrList=new ArrayList<Map.Entry<String, String>>(addrMap.entrySet()); //线程数，加一是因为要创建一个线程下载最后不足400个的图片 int nThreads=(addrList.size()/count)+1; //CountDownLatch countDownLatch = new CountDownLatch(nThreads); try boolean downPic=false; //执行多线程下载图片 downPic=downPic(picPath,addrList,picList,pictureList,nThreads,count); if (downPic) ZipUtil.zipByArray(picList,new File(picPath+synDate+".zip")); return true; catch (Exception e) e.printStackTrace(); return false; /** * 根据url地址下载图片 downPic(picPath,addrList(id,uri的map),picList,pictureList(ConcurrentHashMap),nThreads,count); * @throws InterruptedException */ private boolean downPic(String picPath, List<Map.Entry<String, String>> addrList, Map<String, byte[]> picList, Map<String, InputStream> pictureList, int nThreads, int count)throws IOException, InterruptedException ExecutorService threadPool= Executors.newFixedThreadPool(nThreads); // 创建两个个计数器 CountDownLatch begin=new CountDownLatch(0); CountDownLatch end=new CountDownLatch(nThreads); // 循环创建线程 for (int i = 0; i < nThreads; i++) List<Map.Entry<String, String>>subAddrList=null; // 计算每个线程执行的数据 if ((i + 1) == nThreads) int startIndex = (i * count); int endIndex = addrList.size(); subAddrList = addrList.subList(startIndex, endIndex); else int startIndex = (i * count); int endIndex = (i + 1) * count; subAddrList = addrList.subList(startIndex, endIndex); // 线程类 PicDownload mythead = new PicDownload(picPath,subAddrList,picList,pictureList); // 这里执行线程的方式是调用线程池里的threadPool.execute(mythead)方法。 try threadPool.execute(mythead); catch (Exception e) //记录错误日志 return false; begin.countDown(); end.await(); // 执行完关闭线程池 threadPool.shutdown(); //这里一定要循环直到线程池中所有线程都结束才能往下走，测试时由于没有这一步导致子线程下载图片还没完成，而主线程已经往下走了，导致压缩包内没有图片 //也可以使用CountDownLatch实现 /*while (true) if (threadPool.isTerminated()) System.out.println("所有子线程已结束！"); break; */ return true; /** * 创建文件 * * @param path */ private void creatFile(String path) File file = new File(path); if (!file.exists()) file.mkdirs();
package kun.threadSafe; import java.io.File; import java.io.FileOutputStream; import java.io.InputStream; import java.net.HttpURLConnection; import java.net.URL; import java.util.List; import java.util.Map; import java.util.concurrent.CountDownLatch; class PicDownload implements Runnable //下载图片的地址列表 List<Map.Entry<String, String>> addrList; //装载下载成功的图片列表 Map<String, byte[]> picList; Map<String, InputStream> pictureList; //图片本地存储路径 String picPath; CountDownLatch begin,end; public PicDownload(String picPath, List<Map.Entry<String, String>> addrList, Map<String, byte[]> picList, CountDownLatch begin, CountDownLatch end) this.addrList=addrList; this.picList=picList; this.picPath=picPath; this.begin=begin; this.end=end; @Override public void run() try System.out.println(Thread.currentThread().getName()+"------"+Thread.currentThread().getId()); downPicture(addrList); //System.out.println(countDownLatch.getCount()); begin.await(); catch (Exception e) e.printStackTrace(); finally end.countDown(); //countDownLatch.countDown(); /** *使用容器存储下载的图片字节数组 */ public boolean downPicture(List<Map.Entry<String, String>> addrList) throws Exception InputStream is=null; FileOutputStream fos=null; URL url=null; String fileName=null; String picAddr=null; File pic=null; try for(Map.Entry<String, String> addrEntry:addrList) fileName=addrEntry.getKey(); picAddr=addrEntry.getValue(); //创建Url对象 url=new URL(picAddr); //打开连接，创建java.net.URLConnection对象，该对象没有关闭连接的方法，可以转为它的子类HttpURLConnection调用disconnect方法关闭连接。 //java.net.URLConnection和java.net.HttpURLConnection都有设置超时时间的方法关闭连接 HttpURLConnection uc=(HttpURLConnection)url.openConnection(); is=uc.getInputStream(); //URLConnection获取到的流通过InputStream直接写入字节数组会缺失数据，导致下载的图片不完整，使用org.apache.commons.io.IOUtils.toByteArray(urlconnection.openstream())可以解决 byte[] bytes= IOUtils.toByteArray(is);//new byte[is.available()];获取的字节 //流中数据读入字节数组，读入后，流中数据清空 //is.read(bytes); picList.put(fileName+".jpg",bytes); is.close(); return true; catch (Exception e) e.printStackTrace(); return false; finally if (null!=is) is.close();
package kun.threadSafe; import org.apache.commons.net.ftp.FTPClient; import org.apache.commons.net.ftp.FTPClientConfig; import org.apache.commons.net.ftp.FTPConnectionClosedException; import org.apache.commons.net.ftp.FTPReply; import org.springframework.beans.factory.annotation.Value; import java.io.*; public class FtpUtil private FTPClient ftpClient = null; // ftp服务器地址 private String hostName; // ftp服务器默认端口 public static int defaultport = 21; // 登录名 private String userName; // 登录密码 private String password; // 需要访问的远程目录 private String remoteDir; /** * @param hostName * 主机地址 * @param port * 端口号 * @param userName * 用户名 * @param password * 密码 * @param remoteDir * 默认工作目录 * @param is_zhTimeZone * 是否是中文FTP Server端 * @return * @return */ /** * 新增方法 */ public FtpUtil() boolean is_zhTimeZone = true; this.hostName = hostName; this.userName = userName; this.password = password; this.remoteDir = remoteDir == null ? "" : remoteDir; this.ftpClient = new FTPClient(); if (is_zhTimeZone) this.ftpClient.configure(FtpUtil2.Config()); this.ftpClient.setControlEncoding("GBK"); // 登录 this.login(); // 切换目录 this.changeDir(this.remoteDir); this.setFileType(FTPClient.BINARY_FILE_TYPE); ftpClient.setDefaultPort(Integer.parseInt(port)); public FtpUtil(String hostName, int port, String userName, String password, String remoteDir, boolean is_zhTimeZone) this.hostName = hostName; this.userName = userName; this.password = password; defaultport = port; this.remoteDir = remoteDir == null ? "" : remoteDir; this.ftpClient = new FTPClient(); if (is_zhTimeZone) this.ftpClient.configure(FtpUtil2.Config()); this.ftpClient.setControlEncoding("GBK"); // 登录 this.login(); // 切换目录 this.changeDir(this.remoteDir); this.setFileType(FTPClient.ASCII_FILE_TYPE); ftpClient.setDefaultPort(port); /** * 登录FTP服务器 */ public boolean login() boolean success = false; try ftpClient.connect(this.hostName, defaultport); ftpClient.login(this.userName, this.password); int reply; reply = ftpClient.getReplyCode(); if (!FTPReply.isPositiveCompletion(reply)) ftpClient.disconnect(); return success; catch (FTPConnectionClosedException e) // TODO Auto-generated catch block e.printStackTrace(); catch (IOException e) // TODO Auto-generated catch block e.printStackTrace(); success = true; System.out.println("连接到ftp服务器：" + this.hostName + " 成功..开始登录"); return success; private static FTPClientConfig Config() FTPClientConfig conf = new FTPClientConfig(FTPClientConfig.SYST_UNIX); conf.setRecentDateFormatStr("MM月dd日 HH:mm"); // conf.setRecentDateFormatStr("(YYYY年)?MM月dd日( HH:mm)?"); return conf; /** * 变更工作目录 * * @param remoteDir */ public void changeDir(String remoteDir) try this.remoteDir = remoteDir; ftpClient.changeWorkingDirectory(remoteDir); catch (IOException e) // TODO Auto-generated catch block e.printStackTrace(); System.out.println("变更工作目录为:" + remoteDir); /** * 返回上一级目录(父目录) */ public void toParentDir() try ftpClient.changeToParentDirectory(); catch (IOException e) // TODO Auto-generated catch block e.printStackTrace(); /** * 列出当前工作目录下所有文件 */ public String[] ListAllFiles() String[] names = this.ListFiles("*"); return this.sort(names); /** * 列出指定工作目录下的匹配文件 * * @param dir exp: /cim/ * @param file_regEx 通配符为* */ public String[] ListAllFiles(String dir, String file_regEx) String[] names = this.ListFiles(dir + file_regEx); return this.sort(names); /** * 列出匹配文件 * * @param file_regEx 匹配字符,通配符为* */ public String[] ListFiles(String file_regEx) try /** * FTPFile[] remoteFiles = ftpClient.listFiles(file_regEx); * //System.out.println(remoteFiles.length); String[] name = new * String[remoteFiles.length]; if(remoteFiles != null) for(int * i=0;i<remoteFiles.length;i++) if(remoteFiles[i] == null) * name[i] = ""; else * if(remoteFiles[i].getName()==null||remoteFiles * [i].getName().equals * (".")||remoteFiles[i].getName().equals("..")) name[i] = ""; * else name[i] = remoteFiles[i].getName(); * System.out.println(name[i]); */ ftpClient.enterLocalPassiveMode(); String[] name = ftpClient.listNames(file_regEx); ; if (name == null) return new String[0]; return this.sort(name); catch (Exception e) // TODO Auto-generated catch block e.printStackTrace(); return new String[0]; public void Lists(String reg) try String[] a = ftpClient.listNames(reg); if (a != null) for (String b : a) System.out.println(b); catch (IOException e) // TODO Auto-generated catch block e.printStackTrace(); /** * 设置传输文件的类型[文本文件或者二进制文件] * * @param fileType --BINARY_FILE_TYPE,ASCII_FILE_TYPE */ public void setFileType(int fileType) try ftpClient.setFileType(fileType); catch (IOException e) e.printStackTrace(); /** * 上传文件 * * @param localFilePath --本地文件路径+文件名 * @param newFileName --新的文件名 */ public void uploadFile(String localFilePath, String newFileName) // 上传文件 this.ftpClient.enterLocalPassiveMode();// 被动模式连接 BufferedInputStream buffIn = null; try buffIn = new BufferedInputStream(new FileInputStream(localFilePath)); boolean ifUpload = ftpClient.storeFile(newFileName, buffIn); if (!ifUpload) System.out.println("上传文件失败。。。"); else System.out.println("上传文件成功。。。"); catch (Exception e) e.printStackTrace(); finally try if (buffIn != null) buffIn.close(); catch (Exception e) e.printStackTrace(); /** * 上传文件2 * * @param file --FileInputStream的文件 * @param newFileName --新的文件名 */ public void newUploadFile(FileInputStream file, String newFileName) // 上传文件 this.ftpClient.enterLocalPassiveMode();// 被动模式连接 BufferedInputStream buffIn = null; try buffIn = new BufferedInputStream(file); boolean ifUpload = ftpClient.storeFile(newFileName, buffIn); if (!ifUpload) System.out.println("上传文件失败。。。"); else System.out.println("上传文件成功。。。"); catch (Exception e) e.printStackTrace(); finally try if (buffIn != null) buffIn.close(); catch (Exception e) e.printStackTrace(); /** * 下载文件(单个) * * @param remoteFileName --服务器上的文件名 * @param localFileName --本地文件名 */ public boolean downloadFile(String remoteFileName, String localFileName) this.ftpClient.enterLocalPassiveMode();// 被动模式连接 BufferedOutputStream buffOut = null; try buffOut = new BufferedOutputStream(new FileOutputStream( localFileName)); boolean ifDownload = ftpClient .retrieveFile(remoteFileName, buffOut); if (!ifDownload) System.out.println("下载文件失败。。。"); return false; else System.out.println("下载文件成功。。。"); catch (Exception e) e.printStackTrace(); return false; finally try if (buffOut != null) buffOut.close(); catch (Exception e) e.printStackTrace(); return true; /** * 关闭FTP连接 */ public void close() try if (ftpClient != null) ftpClient.logout(); ftpClient.disconnect(); catch (Exception e) e.printStackTrace(); /** * 冒泡排序字符串(从大到小) */ public String[] sort(String[] str_Array) if (str_Array == null) throw new NullPointerException("The str_Array can not be null!"); String tmp = ""; for (int i = 0; i < str_Array.length; i++) for (int j = 0; j < str_Array.length - i - 1; j++) if (str_Array[j].compareTo(str_Array[j + 1]) < 0) tmp = str_Array[j]; str_Array[j] = str_Array[j + 1]; str_Array[j + 1] = tmp; return str_Array; public static void main(String[] strs) FtpUtil2 FtpUtil2 = new FtpUtil2("192.168.1.112", 20011, "test1", "test1", "/", true); FtpUtil2.downloadFile("test.txt", "d:\\\\test.txt");
package kun.threadSafe; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.util.ArrayList; import java.util.Enumeration; import java.util.List; import java.util.Map; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; import java.util.zip.ZipOutputStream; import org.apache.commons.io.IOUtils; public class ZipUtil /** * 压缩文件 * * @param srcfile File[] 需要压缩的文件列表 * @param zipfile File 压缩后的文件 */ public static OutputStream zipFiles(List<File> srcfile, OutputStream outputStream) byte[] buf = new byte[1024]; try // Create the ZIP file ZipOutputStream out = new ZipOutputStream(outputStream); // Compress the files for (int i = 0; i < srcfile.size(); i++) File file = srcfile.get(i); FileInputStream in = new FileInputStream(file); // Add ZIP entry to output stream. out.putNextEntry(new ZipEntry(file.getName())); // Transfer bytes from the file to the ZIP file int len; while ((len = in.read(buf)) > 0) //System.out.println(len+"=============="); out.write(buf, 0, len); // Complete the entry out.closeEntry(); in.close(); // Complete the ZIP file out.close(); catch (IOException e) log.error("ZipUtil zipFiles exception:"+e); return outputStream; /** * 压缩文件 * * @param srcfile File[] 需要压缩的文件列表 * @param zipfile File 压缩后的文件 */ public static void zipFiles(List<File> srcfile, File zipfile) byte[] buf = new byte[1024]; try // Create the ZIP file ZipOutputStream out = new ZipOutputStream(new FileOutputStream(zipfile)); // Compress the files for (int i = 0; i < srcfile.size(); i++) File file = srcfile.get(i); FileInputStream in = new FileInputStream(file); // Add ZIP entry to output stream. out.putNextEntry(new ZipEntry(file.getName())); // Transfer bytes from the file to the ZIP file int len; while ((len = in.read(buf)) > 0) out.write(buf, 0, len); // Complete the entry out.closeEntry(); in.close(); // Complete the ZIP file out.close(); catch (IOException e) log.error("ZipUtil zipFiles exception:"+e); /** * 压缩文件 * srcfile:key:文件名,value:文件对应的输入流 * @param srcfile * @param zipfile * @see */ public static void zipByStream(Map<String,InputStream> srcfile, File zipfile) try // Create the ZIP file ZipOutputStream out = new ZipOutputStream(new FileOutputStream(zipfile)); // Compress the files System.out.println(srcfile.entrySet().size()); for (Map.Entry<String, InputStream> fileEntry:srcfile.entrySet()) InputStream in = fileEntry.getValue(); // Add ZIP entry to output stream. System.out.println(in.available()); out.putNextEntry(new ZipEntry(fileEntry.getKey())); // Transfer bytes from the file to the ZIP file byte[] bytes=IOUtils.toByteArray(in); out.write(bytes); out.closeEntry(); in.close(); // Complete the ZIP file out.close(); catch (IOException e) log.error("ZipUtil zipFiles exception:"+e); System.out.println(e.getMessage()); /** * 压缩文件 * srcfile:key:文件名,value:文件对应的字节数组 * @param srcfile * @param zipfile * @see */ public static void zipByArray(Map<String,byte[]> srcfile, File zipfile) byte[] buf = new byte[1024]; try // Create the ZIP file ZipOutputStream out = new ZipOutputStream(new FileOutputStream(zipfile)); // Compress the files System.out.println(srcfile.entrySet().size()); for (Map.Entry<String, byte[]> fileEntry:srcfile.entrySet()) //InputStream in = fileEntry.getValue(); // Add ZIP entry to output stream. out.putNextEntry(new ZipEntry(fileEntry.getKey())); // Transfer bytes from the file to the ZIP file byte[] bytes=fileEntry.getValue();//IOUtils.toByteArray(in); out.write(bytes); out.closeEntry(); //in.close(); // Complete the ZIP file out.close(); catch (IOException e) log.error("ZipUtil zipFiles exception:"+e); System.out.println(e.getMessage()); /** * 解压缩 * * @param zipfile File 需要解压缩的文件 * @param descDir String 解压后的目标目录 */ public static void unZipFiles(File zipfile, String descDir) try // Open the ZIP file ZipFile zf = new ZipFile(zipfile); for (Enumeration entries = zf.entries(); entries.hasMoreElements();) // Get the entry name ZipEntry entry = ((ZipEntry) entries.nextElement()); String zipEntryName = entry.getName(); InputStream in = zf.getInputStream(entry); // System.out.println(zipEntryName); OutputStream out = new FileOutputStream(descDir + zipEntryName); byte[] buf1 = new byte[1024]; int len; while ((len = in.read(buf1)) > 0) out.write(buf1, 0, len); // Close the file and stream in.close(); out.close(); catch (IOException e) log.error("ZipUtil unZipFiles exception:"+e); /** * Main * * @param args */ public static void main(String[] args) List<File> srcfile=new ArrayList<File>(); srcfile.add(new File("d:\\\\1.jpg")); srcfile.add(new File("d:\\\\2.jpg")); srcfile.add(new File("d:\\\\3.jpg")); srcfile.add(new File("d:\\\\4.jpg")); File zipfile = new File("d:\\\\pic.zip"); ZipUtil.zipFiles(srcfile, zipfile);
Java多线程断点下载文件并压缩
引言：使用多线程下载图片可以大幅度的提升下载速度，基于封装理念把可以抽离出来的代码全部抽离。
代码思路：就是先把需要下载的文件放入一个集合，再下载到一个临时目录，在压缩，最后删除临时目录中的下载文件。
下载
设置Header
import java.util.HashMap; /** * @author XuYanK * @date 2020/5/18 9:08 */ public class Header { public static final HashMap<String ,String> header = new HashMap<>(); private Header() { } static { header.put("Accept-Encoding", "gzip, deflate"); header.put("Accept-Language", "zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2"); header.put("Cache-Control", "max-age=0"); header.put("Connection", "keep-alive"); header.put("User-Agent","Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:59.0) Gecko/20100101 Firefox/59.0"); header.put("Accept", "image/gif,image/jpeg,image/pjpeg,image/pjpeg, " + "application/x-shockwave-flash, application/xaml+xml, " + "application/vnd.ms-xpsdocument, application/x-ms-xbap" + "application/x-ms-application,application/vnd.ms-excel" + "application/vnd.ms-powerpoint, application/msword,*/*"); } }
编写下载线程类 DowloadThred
import java.io.InputStream; import java.io.RandomAccessFile; import java.net.HttpURLConnection; import java.net.URL; import java.util.concurrent.CountDownLatch; /** * @author XuYanK * @date 2020/5/18 9:10 */ public class DownloadThread implements Runnable { //当前线程的下载位置 private int startPos; //定义当前线程负责下载的文件大小 private int currentPartSize; //当前线程需要下载的文件块,此类的实例支持对随机访问文件的读取和写入。 private RandomAccessFile currentPart; //定义该线程已下载的字节数 private int length; //线程计数器 private CountDownLatch latch; private String path; public int getLength() { return length; } public DownloadThread(int startPos,int currentPartSize,RandomAccessFile currentPart, String path ,CountDownLatch latch){ this.startPos = startPos; this.currentPartSize = currentPartSize; this.currentPart = currentPart; this.path = path; this.latch =latch; } @Override public void run(){ InputStream inputStream = null; try{ URL url = new URL(path); HttpURLConnection conn = (HttpURLConnection)url.openConnection(); conn.setConnectTimeout(5 * 1000); //设置请求方法 conn.setRequestMethod("GET"); //设置请求属性 Header.header.forEach((key, value) -> conn.setRequestProperty(key,value)); inputStream = conn.getInputStream(); //inputStream.skip(n);跳过和丢弃此输入流中数据的 n 个字节 inputStream.skip(this.startPos); byte[] buffer = new byte[1024]; int hasRead = 0; //读取网络数据写入本地 while(length < currentPartSize && (hasRead = inputStream.read(buffer)) != -1){ currentPart.write(buffer, 0, hasRead); length += hasRead; } } catch(Exception e){ e.printStackTrace(); }finally { try { currentPart.close(); inputStream.close(); }catch (Exception e){ e.printStackTrace(); } latch.countDown(); } } }
编写下载工具类 DownUtil
import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.RandomAccessFile; import java.math.BigDecimal; import java.net.HttpURLConnection; import java.net.URL; import java.util.concurrent.CountDownLatch; /** * @author XuYanK * @date 2020/5/18 9:10 */ public class DownUtil { private Logger logger = LoggerFactory.getLogger(DownUtil.class); //定义下载路径 private String path; //指定所下载的文件的保存位置 private String targetFile; //定义下载线程的数量 private int threadNum; //定义下载线程的对象 private DownloadThread[] threads; //下载文件的总大小 private int fileSize; //线程计数器 private CountDownLatch latch; public DownUtil(String path, String targetFile, int threadNum){ this.path = path; this.targetFile = targetFile; this.threadNum = threadNum; threads = new DownloadThread[threadNum]; latch = new CountDownLatch(threadNum); } /** * 多线程下载文件 * @throws Exception */ public void downLoad() throws Exception{ long t1 = System.currentTimeMillis(); URL url = new URL(path); HttpURLConnection conn = (HttpURLConnection)url.openConnection(); conn.setConnectTimeout(5 * 1000); //设置请求方法 // conn.setRequestMethod("POST"); //设置请求属性 Header.header.forEach((key, value) -> conn.setRequestProperty(key,value)); //得到文件大小 fileSize = conn.getContentLength(); conn.disconnect(); int currentPartSize = fileSize / threadNum + 1; RandomAccessFile file = new RandomAccessFile(targetFile,"rw"); //设置本地文件大小 file.setLength(fileSize); file.close(); for(int i = 0;i < threadNum;i++){ //计算每个线程的下载位置 int startPos = i * currentPartSize; //每个线程使用一个RandomAccessFile进行下载 RandomAccessFile currentPart = new RandomAccessFile(targetFile,"rw"); //定位该线程的下载位置 currentPart.seek(startPos); //创建下载线程 threads[i] = new DownloadThread(startPos, currentPartSize, currentPart , path , latch); Thread t = new Thread(threads[i]); t.start(); } BigDecimal n = BigDecimal.ZERO; while (true){ BigDecimal length = BigDecimal.ZERO; if (latch.getCount()==0 && BigDecimal.ONE.compareTo(n)<=0){break;} for (DownloadThread downloadThread:threads) { length = length.add(new BigDecimal(downloadThread.getLength())); } n = length.divide(new BigDecimal(fileSize), 4, BigDecimal.ROUND_HALF_UP); logger.info("已下载"+n.multiply(new BigDecimal("100"))+"%"); Thread.sleep(1000); } long t2 = System.currentTimeMillis(); logger.info("下载完成，文件大小为"+fileSize*1.0/(1024*1024)+"M，共用时"+(t2-t1)*1.0/1000+"秒"); //线程等待,一个文件下载成功后继续下一个 latch.await(); } /** * 根据文件名获取文件类型 * @param fileName * @return */ public static String getFileType(String fileName) { return fileName.substring(fileName.lastIndexOf(".")); } }
压缩
压缩工具类ZipUtils
import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.*; import java.util.ArrayList; import java.util.List; import java.util.zip.CRC32; import java.util.zip.CheckedOutputStream; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; /** * @author XuYanK * @date 2020/5/18 9:45 */ public class ZipUtils { private static Logger logger = LoggerFactory.getLogger(ZipUtils.class); /** 缓冲器大小 */ private static final int BUFFER = 512; /**压缩得到的文件的后缀名*/ private static final String SUFFIX=".zip"; /** *文件写入压缩包 * @param sb * @param zipname 压缩包名称 * @throws IOException */ public static void writeZip(StringBuffer sb, String zipname) throws IOException { String[] files = sb.toString().split(","); OutputStream os = new BufferedOutputStream(new FileOutputStream(zipname + SUFFIX)); ZipOutputStream zos = new ZipOutputStream(os); byte[] buf = new byte[8192]; int len; for (int i = 0; i < files.length; i++) { File file = new File(files[i]); if (!file.isFile()) { continue; } ZipEntry ze = new ZipEntry(file.getName()); zos.putNextEntry(ze); BufferedInputStream bis = new BufferedInputStream(new FileInputStream(file)); while ((len = bis.read(buf)) > 0) { zos.write(buf, 0, len); } zos.closeEntry(); } zos.closeEntry(); zos.close(); } /** * 得到源文件路径的所有文件 * @param dirFile 压缩源文件路径 * */ public static List<File> getAllFile(File dirFile){ List<File> fileList=new ArrayList<>(); File[] files= dirFile.listFiles(); //文件 for(File file:files){ if(file.isFile()){ fileList.add(file); logger.info("文件添加到压缩列表:"+file.getName()); }else {//目录 //非空目录 if(file.listFiles().length!=0){ //把递归文件加到fileList中 fileList.addAll(getAllFile(file)); }else {//空目录 fileList.add(file); logger.info("空目录添加到压缩列表:"+file.getName()); } } } return fileList; } /** * 压缩文件 * @param dirPath 压缩源文件路径 * @param zipFileName 压缩目标文件路径 * */ public static void compress(String dirPath,String zipFileName) { //添加文件的后缀名 zipFileName = zipFileName + SUFFIX; File dirFile = new File(dirPath); List<File> fileList = getAllFile(dirFile); byte[] buffer = new byte[BUFFER]; ZipEntry zipEntry = null; //每次读取出来的长度 int readLength = 0; try { // 对输出文件做CRC32校验 CheckedOutputStream cos = new CheckedOutputStream(new FileOutputStream( zipFileName), new CRC32()); ZipOutputStream zos = new ZipOutputStream(cos); for (File file : fileList) { //若是文件，则压缩文件 if (file.isFile()) { zipEntry = new ZipEntry(getRelativePath(dirPath, file)); zipEntry.setSize(file.length()); zipEntry.setTime(file.lastModified()); zos.putNextEntry(zipEntry); InputStream is = new BufferedInputStream(new FileInputStream(file)); while ((readLength = is.read(buffer, 0, BUFFER)) != -1) { zos.write(buffer, 0, readLength); } is.close(); logger.info("file compress:" + file.getCanonicalPath()); System.out.println(); } else { //若是空目录，则写入zip条目中 zipEntry = new ZipEntry(getRelativePath(dirPath, file)); zos.putNextEntry(zipEntry); logger.info("dir compress: " + file.getCanonicalPath() + "/"); } } zos.close(); //最后得关闭流，不然压缩最后一个文件会出错 } catch (FileNotFoundException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } } /** * 获取相对路径 * @param dirPath 源文件路径 * @param file 准备压缩的单个文件 * */ public static String getRelativePath(String dirPath,File file){ File dirFile=new File(dirPath); String relativePath=file.getName(); while (true){ file=file.getParentFile(); if(file==null) break; if(file.equals(dirFile)){ break; } else { relativePath=file.getName()+"/"+relativePath; } } return relativePath; } /** * 删除文件夹目录下的文件 * @param path * @return */ public static boolean deleteDir(String path) { File file = new File(path); //判断是否待删除目录是否存在 if (!file.exists()) { logger.warn("文件目录不存在"); return false; } //取得当前目录下所有文件和文件夹 String[] content = file.list(); for (String name : content) { File temp = new File(path, name); //判断是否是目录 if (temp.isDirectory()) { //递归调用，删除目录里的内容 deleteDir(temp.getAbsolutePath()); //删除空目录 temp.delete(); } else { //直接删除文件 if (!temp.delete()) { logger.warn("删不掉，使用强制删除方法GC()"); //强制删除 // System.gc(); // temp.delete(); } } } return true; } }
测试
main方法
import com.alibaba.fastjson.JSONObject; import lombok.extern.slf4j.Slf4j; import java.io.File; import java.util.List; import java.util.Map; /** * @author XuYanK * @date 2020/5/18 9:13 */ @Slf4j public class MyDowloadTest { public static void main(String[] args) { //系统本地临时文件夹路径 String targetFlie =System.getProperty("java.io.tmpdir")+File.separator+"testDown"; //判断是否存在该文件夹 File file = new File(targetFlie); if (!file.exists()) { file.mkdirs(); } String listData = "[{ " + " "userId": "1", " + " "username": "admin", " + " "password": "123456", " + " "imgpath": "http://c.hiphotos.baidu.com/zhidao/pic/item/54fbb2fb43166d22d552b941432309f79052d23a.jpg" " + " }, " + " { " + " "userId": "2", " + " "username": "jacks", " + " "password": "111111", " + " "imgpath": "http://e.hiphotos.baidu.com/zhidao/pic/item/b8014a90f603738d8d7adbbbb31bb051f819ec73.jpg" " + " } " + "]"; //转换list对象 List<Map> lists = JSONObject.parseArray(listData, Map.class); for (int i = 0; i < lists.size(); i++) { String imgpath = (String) lists.get(i).get("imgpath"); //获取文件后缀 String fileType = DownUtil.getFileType(imgpath); //多线程下载列表图片 DownUtil downUtil = new DownUtil( imgpath, targetFlie + "\" + System.currentTimeMillis() + fileType, 8); try { downUtil.downLoad(); } catch (Exception e) { e.printStackTrace(); } } String zipName = "测试"; ZipUtils.compress(targetFlie, zipName); ZipUtils.deleteDir(targetFlie); // FileUtils.deleteQuietly(new File(targetFlie)); } }
以上是关于多线程下载图片并压缩的主要内容，如果未能解决你的问题，请参考以下文章
手把手教你线程池配合CompletableFuture实现图片下载并压缩