多线程httpWebRequest

Posted 2020-08-16 whoareyou

//在初始化方法设置以下相关属性.              
                //对象所允许的最大并发连接数//可在配置文件中设置
                System.Net.ServicePointManager.DefaultConnectionLimit = 1000;
                //是否使用 Nagle 不使用 提高效率 
                System.Net.ServicePointManager.UseNagleAlgorithm = false;
                //对象的最大空闲时间.(默认为100秒的)
                System.Net.ServicePointManager.MaxServicePointIdleTime = 3600 * 1000;
//-------------------------------------------------------------------------------------------
//在 while循环中的处理..
 while (!worker.CancellationPending)
{
                request = WebRequest.Create(userUrl) as HttpWebRequest;
                 request.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip,deflate");
                request.Proxy = null;//
                request.KeepAlive = true;
                //数据是否缓冲 false 提高效率  
                request.AllowWriteStreamBuffering = false;

                 Stream stream = null;
                if (response.ContentEncoding.ToLower().Contains("gzip"))
                {
                    stream = new GZipStream(response.GetResponseStream(), CompressionMode.Decompress);
                }
                else if (response.ContentEncoding.ToLower().Contains("deflate"))
                {
                    stream = new DeflateStream(response.GetResponseStream(), CompressionMode.Decompress);
                }
                else
                {
                    stream = response.GetResponseStream();
                }
                using (stream)
                {
                    using (StreamReader sr = new StreamReader(stream, true))
                    {
                        string resultString = sr.ReadToEnd();
                    }

//-------------------------------------------------------------------------------------------

//使用完后要进行释放，否则易造成网络堵塞
                if (request != null)
                {
                    request.Abort();
                    request = null;
                }
                if (response != null)
                {
                    response.Close();
                    response = null;
                }
}

//在适当时间进行 强制垃圾回收.
System.GC.Collect();

//-------------------------------------------------------------------------------------------

//以上是总结的关于多线程使用 HttpWebRequest 用于提高效率获取网页源代码的 相关设置. (省略了其他代码..)