unity-多线程断点下载HttpWebRequest

Posted 2022-07-13 蝶泳奈何桥.

---

title: unity-多线程断点下载HttpWebRequest
categories: Unity3d
tags: [unity, 多线程, 断点, 下载]
date: 2022-07-08 14:15:00
comments: false
mathjax: true
toc: true

unity-多线程断点下载HttpWebRequest

---

前篇

在 unity-多线程异步下载HttpWebRequest 的基础上, 增加断点下载功能 (代码有稍作修改, 适配断点下载), 应用场景是下载一个大文件时, 将文件分割成多个片段进行下载, 即使中间断网后, 重新下载时重已累积的下载大小的基础上, 继续剩余未下载完内容.

效果如下, 最终下载完大小是 31,715KB (中间鼠标框选时, 进程已经杀了, 中断了下载, 后面重启进程继续累计下载)

---

代码

• 断点下载 MultiResumeMgr.cs

  using System;
  using System.Collections;
  using System.Collections.Generic;
  using System.IO;
  using System.Linq;
  using System.Net;
  using System.Text;
  using LuaInterface;
  using UnityEngine;
  
  // 多线程断点下载
  public class MultiResumeMgr : MonoBehaviour 
      private static MultiResumeMgr _instance;
      public static MultiResumeMgr Instance 
          get  return _instance; 
      
  
      private List<MultiResumeObj> mObjList = new List<MultiResumeObj>();
  
      public MultiDownMgr multiDownIns;
      public int chunkMinSize = 1024 * 1024; // 分块最小值
      public int mergeBufferSize = 1024 * 1024;
  
      void Awake() 
          _instance = this;
      
  
      void Start() 
          if (multiDownIns == null) 
              multiDownIns = gameObject.GetComponent<MultiDownMgr>();
              if (multiDownIns == null) 
                  multiDownIns = gameObject.AddComponent<MultiDownMgr>();
                  multiDownIns.bufferSize = chunkMinSize;
              
          
      
  
      long GetContentLength(string url) 
          try 
              var req = (HttpWebRequest) WebRequest.CreateDefault(new Uri(url));
              req.Method = "HEAD";
              req.Timeout = 5000;
              var res = (HttpWebResponse) req.GetResponse();
              long contentLen = 0;
              if (res.StatusCode == HttpStatusCode.OK) 
                  contentLen = res.ContentLength;
              
  
              res.Close();
              req.Abort();
              return contentLen;
           catch (System.Exception ex) 
              return 0;
          
      
  
      public void Request(string url, string path, LuaFunction completeFn, LuaFunction progressFn, int chunkNum = 10) 
          MultiResumeObj msObj = GetMultiResumeObj(url, path, chunkNum);
          msObj.luaFn = completeFn;
          msObj.progressFn = progressFn;
          RequestObj(msObj);
      
  
      [NoToLua]
      public void RequestObj(MultiResumeObj msObj) 
          mObjList.Add(msObj);
          StartCoroutine(StartTask(msObj));
      
  
      // 获取分块信息
      private MultiResumeObj GetMultiResumeObj(string url, string path, int chunkNum) 
          long contentLen = GetContentLength(url);
          // LogUtil.D("--- contentLen: 0", contentLen);
  
          MultiResumeObj msObj = new MultiResumeObj();
          msObj.url = url;
          msObj.path = path;
          msObj.contentLen = contentLen;
          msObj.lenFile = string.Format("0.len", path);;
          msObj.chunkLst = new List<MultiDownObj>();
  
          // 小于 最小值
          if (contentLen < chunkMinSize || chunkNum < 2) 
              MultiDownObj mdObj = new MultiDownObj();
              mdObj.url = url;
              mdObj.path = path;
              mdObj.from = 0;
              mdObj.to = contentLen - 1;
              mdObj.order = 0;
              msObj.chunkLst.Add(mdObj);
           else 
              // 计算分块大小
              long chunkSize = contentLen / chunkNum + 1;
              // LogUtil.D("--- chunkSize: 0", chunkSize);
              long offset = 0;
              for (int i = 0; i < chunkNum; i++) 
                  long from = offset;
                  long to = i == chunkNum - 1 ? contentLen - 1 : offset + chunkSize; // 最后一块大小直接读到 end
                  offset = to + 1; // 下一片段的开始值偏移量
                  string partFile = string.Format("0.part1:00", path, i + 1);
                  // LogUtil.D("--- idx: 0, from: 1, to: 2, path: 3", i, from, to, partFile);
                  MultiDownObj mdObj = new MultiDownObj();
                  mdObj.url = url;
                  mdObj.path = partFile;
                  mdObj.from = from;
                  mdObj.to = to;
                  mdObj.order = i;
  
                  msObj.chunkLst.Add(mdObj);
              
          
          return msObj;
      
  
      // 开始任务
      IEnumerator StartTask(MultiResumeObj msObj) 
          yield return null;
  
          // 判断要下载的文件以缓存文件的记录长度是否一致, 不一致要清楚 part
          CacheDiffDeal(msObj);
  
          SortedList<int, MultiDownObj> sortLst = new SortedList<int, MultiDownObj>();
          foreach (MultiDownObj mdObj in msObj.chunkLst) 
              sortLst.Add(mdObj.order, mdObj);
          
  
          int cnt = 0;
          // 此回调已经是主线程, 可以打 log
          MultiDownDlg onCompleteFn = (MultiDownObj mdObj) => 
              cnt++;
  
              // 下载进度通知 lua
              if (msObj.progressFn != null) 
                  msObj.progressFn.Call(cnt, msObj.chunkLst.Count);
              
  
              // 只有一块, 不需要合并文件
              if (msObj.chunkLst.Count == 1) 
                  Close(msObj);
                  msObj.isDone = true;
                  return;
              
  
              if (cnt == sortLst.Count)  // 全部下载完成
                  // 是否全部 success 
                  bool isAllSucc = true;
                  string partErr = "";
                  foreach (var item in sortLst) 
                      if (!(item.Value.code == (int) HttpStatusCode.OK ||
                              item.Value.code == (int) HttpStatusCode.PartialContent)) 
                          isAllSucc = false;
                          partErr += string.Format("\\n order: 0, code: 1, msg: 1", item.Value.order, item.Value.code, item.Value.path);
                      
                  
  
                  if (isAllSucc) 
                      // LogUtil.D("--- all success");
                      msObj.code = (int) HttpStatusCode.PartialContent;
                      List<string> fileLst = sortLst.Select((item) =>  return item.Value.path; ).ToList();
                      Close(msObj); // 必须先关闭文件流, 才能进行读取合并
  
                      string cbErr = Combine(fileLst, msObj.path);
                      if (cbErr != null)  // 合并文件失败
                          LogUtil.E("--- combine err: 0", cbErr);
                          msObj.code = (int) EMSErr.Combine;
                          msObj.path = partErr;
                      
  
                      RemoveParts(msObj);
                   else 
                      LogUtil.E("--- some part err: 0", partErr);
                      msObj.code = (int) EMSErr.DownPart;
                      msObj.path = partErr;
                  
  
                  msObj.isDone = true;
              
          ;
  
          foreach (MultiDownObj mdObj in msObj.chunkLst) 
              FileStream fs = new FileStream(mdObj.path, FileMode.Append, FileAccess.Write);
              mdObj.fs = fs;
              mdObj.csFn = onCompleteFn;
  
              long rest = (mdObj.to - mdObj.from + 1) - fs.Length;
              // LogUtil.D("--- order: 0, rest: 1, path: 2", mdObj.order, rest, mdObj.path);
              if (rest > 0) 
                  mdObj.from = mdObj.to - rest + 1; // 重新计算下载开始值
                  multiDownIns.RequestObj(mdObj);
               else 
                  // 无需再下载, 直接回调
                  mdObj.code = (int) HttpStatusCode.PartialContent;
                  mdObj.csFn(mdObj);
              
          
      
  
      string Combine(List<string> fileLst, string outPath) 
          Utils.DeleteFile(outPath);
          byte[] buffer = new byte[mergeBufferSize];
          using(FileStream outStream = new FileStream(outPath, FileMode.Create)) 
              int readLen = 0;
              FileStream srcStream = null;
              try 
                  for (int i = 0; i < fileLst.Count; i++) 
                      srcStream = new FileStream(fileLst[i], FileMode.Open);
                      while ((readLen = srcStream.Read(buffer, 0, mergeBufferSize)) > 0) 
                          outStream.Write(buffer, 0, readLen);
                          outStream.Flush();
                      
                      srcStream.Close();
                  
               catch (System.Exception ex) 
                  if (srcStream != null) 
                      srcStream.Close();
                  
                  return ex.Message;
              
          
          return null;
      
  
      /*
      // 合并文件流, fsLst 必须是顺序的, 没必要写成异步
      // FileMode.Append 模式必须要要求是 FileAccess.Write 权限, 而 FileAccess.Write 权限又不能 read, 所以没法直接把 fileStream 合并, 所以这个方法不能使用
      void Combine(List<FileStream> fsLst, string outPath) 
          Utils.DeleteFile(outPath);
          byte[] buffer = new byte[mergeBufferSize];
          using(FileStream outStream = new FileStream(outPath, FileMode.Create)) 
              int readLen = 0;
              for (int i = 0; i < fsLst.Count; i++) 
                  FileStream srcStream = fsLst[i];
                  while ((readLen = srcStream.Read(buffer, 0, mergeBufferSize)) > 0) 
                      outStream.Write(buffer, 0, readLen);
                      outStream.Flush();
                  
              
          
      
      */
  
      // 缓存文件长度记录不一致, 删掉
      void CacheDiffDeal(MultiResumeObj msObj) 
          if (msObj.lenFile != null) 
              string txt = Utils.ReadAllTextFromFile(msObj.lenFile);
              if (txt != null) 
                  long cacheLen = long.Parse(txt);
                  if (msObj.contentLen != cacheLen) 
                      // LogUtil.D("--- cache len diff, dstLen: 0, cacheLen: 1", msObj.contentLen, cacheLen);
                      RemoveParts(msObj);
                  
              
          
          Utils.WriteFileUTF8(msObj.lenFile, msObj.contentLen.ToString());
      
  
      // 删除临时文件
      void RemoveParts(MultiResumeObj msObj) 
          Utils.DeleteFile(msObj.lenFile);
          if (msObj != null && msObj.chunkLst != null) 
              foreach (MultiDownObj mdObj in msObj.chunkLst) 
                  Utils.DeleteFile(mdObj.path);
              
          
      
  
      // 释放资源
      void Close(MultiResumeObj msObj) 
          if (msObj == null)
              return;
  
          msObj.isActive = false;
  
          // 释放文件流
          if (msObj.chunkLst != null) 
              foreach (var mdObj in msObj.chunkLst) 
                  mdObj.isActive = false;
                  if (mdObj.fs != null) 
                      mdObj.fs.Close();
                      mdObj.fs = null;
                  
              
          
  
          if (msObj.progressFn != null) 
              msObj.progressFn.Dispose();
              msObj.progressFn = null;
          
      
  
      public void StopDown(string url) 
          for (int i = 0; i < mObjList.Count; ++i) 
              MultiResumeObj msObj = mObjList[i];
              Close(msObj);
  
              if (msObj.url == url) 
                  mObjList.RemoveAt(i);
                  i -= 1;
              
          
      
  
      public void StopAll() 
          for (int i = 0; i < mObjList.Count; ++i) 
              MultiResumeObj msObj = mObjList[i];
              StopDown(msObj.url);
          
          mObjList.Clear();
      
  
      void OnDestroy() 
          StopAll();
      
  
      void Update() 
          for (int i = 0; i < mObjList.Count; ++i) 
              MultiResumeObj msObj = mObjList[i];
              if (msObj.isDone) 
                  // LogUtil.D("--- msObj done, code: 0, path: 1", msObj.code, msObj.path);
                  if (msObj.luaFn != null) 
                      msObj.luaFn.Call(msObj.code, msObj.url, msObj.path);
                      msObj.luaFn.Dispose();
                      msObj.luaFn = null;
                  
  
                  if (msObj.csFn != null) 
                      msObj.csFn(msObj);
                      msObj.csFn = null;
                  
  
                  Close(msObj);
                  mObjList.RemoveAt(i);
                  i -= 1;
              
          
      
  
     
  以上是关于unity-多线程断点下载HttpWebRequest的主要内容，如果未能解决你的问题，请参考以下文章
   
   unity-多线程断点下载HttpWebRequest
   unity-多线程断点下载HttpWebRequest
   JAVA下实现多线程断点下载
   java多线程断点下载原理（代码实例演示）
   iOS开发网络篇—多线程断点下载
   多线程下载文件（支持暂停取消断点续传）