HttpHelper工具类

Posted dangzhenjiuhao

tags:

篇首语:本文由小常识网(cha138.com)小编为大家整理,主要介绍了HttpHelper工具类相关的知识,希望对你有一定的参考价值。

  1. /// <summary>  
  2. /// 类说明:HttpHelper类,用来实现Http访问,Post或者Get方式的,直接访问,带Cookie的,带证书的等方式,可以设置代理  
  3. /// 重要提示:请不要自行修改本类,如果因为你自己修改后将无法升级到新版本。如果确实有什么问题请到官方网站提建议,  
  4. /// 我们一定会及时修改  
  5. /// 编码日期:2011-09-20  
  6. /// 编 码 人:苏飞  
  7. /// 联系方式:361983679    
  8. /// 官方网址:http://www.sufeinet.com/thread-3-1-1.html  
  9. /// 修改日期:2015-03-25  
  10. /// 版 本 号:1.4.7  
  11. /// </summary>  
  12. using System;  
  13. using System.Collections.Generic;  
  14. using System.Text;  
  15. using System.Net;  
  16. using System.IO;  
  17. using System.Text.RegularExpressions;  
  18. using System.IO.Compression;  
  19. using System.Security.Cryptography.X509Certificates;  
  20. using System.Net.Security;  
  21.   
  22. namespace DotNet.Utilities  
  23. {  
  24.     /// <summary>  
  25.     /// Http连接操作帮助类  
  26.     /// </summary>  
  27.     public class HttpHelper  
  28.     {  
  29.  
  30.         #region 预定义方变量  
  31.         //默认的编码  
  32.         private Encoding encoding = Encoding.Default;  
  33.         //Post数据编码  
  34.         private Encoding postencoding = Encoding.Default;  
  35.         //HttpWebRequest对象用来发起请求  
  36.         private HttpWebRequest request = null;  
  37.         //获取影响流的数据对象  
  38.         private HttpWebResponse response = null;  
  39.         #endregion  
  40.  
  41.         #region Public  
  42.   
  43.         /// <summary>  
  44.         /// 根据相传入的数据,得到相应页面数据  
  45.         /// </summary>  
  46.         /// <param name="item">参数类对象</param>  
  47.         /// <returns>返回HttpResult类型</returns>  
  48.         public HttpResult GetHtml(HttpItem item)  
  49.         {  
  50.             //返回参数  
  51.             HttpResult result = new HttpResult();  
  52.             try  
  53.             {  
  54.                 //准备参数  
  55.                 SetRequest(item);  
  56.             }  
  57.             catch (Exception ex)  
  58.             {  
  59.                 result.Cookie = string.Empty;  
  60.                 result.Header = null;  
  61.                 result.Html = ex.Message;  
  62.                 result.StatusDescription = "配置参数时出错:" + ex.Message;  
  63.                 //配置参数时出错  
  64.                 return result;  
  65.             }  
  66.             try  
  67.             {  
  68.                 //请求数据  
  69.                 using (response = (HttpWebResponse)request.GetResponse())  
  70.                 {  
  71.                     GetData(item, result);  
  72.                 }  
  73.             }  
  74.             catch (WebException ex)  
  75.             {  
  76.                 if (ex.Response != null)  
  77.                 {  
  78.                     using (response = (HttpWebResponse)ex.Response)  
  79.                     {  
  80.                         GetData(item, result);  
  81.                     }  
  82.                 }  
  83.                 else  
  84.                 {  
  85.                     result.Html = ex.Message;  
  86.                 }  
  87.             }  
  88.             catch (Exception ex)  
  89.             {  
  90.                 result.Html = ex.Message;  
  91.             }  
  92.             if (item.IsToLower) result.Html = result.Html.ToLower();  
  93.             return result;  
  94.         }  
  95.         #endregion  
  96.  
  97.         #region GetData  
  98.   
  99.         /// <summary>  
  100.         /// 获取数据的并解析的方法  
  101.         /// </summary>  
  102.         /// <param name="item"></param>  
  103.         /// <param name="result"></param>  
  104.         private void GetData(HttpItem item, HttpResult result)  
  105.         {  
  106.             #region base  
  107.             //获取StatusCode  
  108.             result.StatusCode = response.StatusCode;  
  109.             //获取StatusDescription  
  110.             result.StatusDescription = response.StatusDescription;  
  111.             //获取Headers  
  112.             result.Header = response.Headers;  
  113.             //获取CookieCollection  
  114.             if (response.Cookies != null) result.CookieCollection = response.Cookies;  
  115.             //获取set-cookie  
  116.             if (response.Headers["set-cookie"] != null) result.Cookie = response.Headers["set-cookie"];  
  117.             #endregion  
  118.  
  119.             #region byte  
  120.             //处理网页Byte  
  121.             byte[] ResponseByte = GetByte();  
  122.             #endregion  
  123.  
  124.             #region Html  
  125.             if (ResponseByte != null & ResponseByte.Length > 0)  
  126.             {  
  127.                 //设置编码  
  128.                 SetEncoding(item, result, ResponseByte);  
  129.                 //得到返回的HTML  
  130.                 result.Html = encoding.GetString(ResponseByte);  
  131.             }  
  132.             else  
  133.             {  
  134.                 //没有返回任何Html代码  
  135.                 result.Html = string.Empty;  
  136.             }  
  137.             #endregion  
  138.         }  
  139.         /// <summary>  
  140.         /// 设置编码  
  141.         /// </summary>  
  142.         /// <param name="item">HttpItem</param>  
  143.         /// <param name="result">HttpResult</param>  
  144.         /// <param name="ResponseByte">byte[]</param>  
  145.         private void SetEncoding(HttpItem item, HttpResult result, byte[] ResponseByte)  
  146.         {  
  147.             //是否返回Byte类型数据  
  148.             if (item.ResultType == ResultType.Byte) result.ResultByte = ResponseByte;  
  149.             //从这里开始我们要无视编码了  
  150.             if (encoding == null)  
  151.             {  
  152.                 Match meta = Regex.Match(Encoding.Default.GetString(ResponseByte), "<meta[^<]*charset=([^<]*)[\"‘]", RegexOptions.IgnoreCase);  
  153.                 string c = string.Empty;  
  154.                 if (meta != null && meta.Groups.Count > 0)  
  155.                 {  
  156.                     c = meta.Groups[1].Value.ToLower().Trim();  
  157.                 }  
  158.                 if (c.Length > 2)  
  159.                 {  
  160.                     try  
  161.                     {  
  162.                         encoding = Encoding.GetEncoding(c.Replace("\"", string.Empty).Replace("‘", "").Replace(";", "").Replace("iso-8859-1", "gbk").Trim());  
  163.                     }  
  164.                     catch  
  165.                     {  
  166.                         if (string.IsNullOrEmpty(response.CharacterSet))  
  167.                         {  
  168.                             encoding = Encoding.UTF8;  
  169.                         }  
  170.                         else  
  171.                         {  
  172.                             encoding = Encoding.GetEncoding(response.CharacterSet);  
  173.                         }  
  174.                     }  
  175.                 }  
  176.                 else  
  177.                 {  
  178.                     if (string.IsNullOrEmpty(response.CharacterSet))  
  179.                     {  
  180.                         encoding = Encoding.UTF8;  
  181.                     }  
  182.                     else  
  183.                     {  
  184.                         encoding = Encoding.GetEncoding(response.CharacterSet);  
  185.                     }  
  186.                 }  
  187.             }  
  188.         }  
  189.         /// <summary>  
  190.         /// 提取网页Byte  
  191.         /// </summary>  
  192.         /// <returns></returns>  
  193.         private byte[] GetByte()  
  194.         {  
  195.             byte[] ResponseByte = null;  
  196.             MemoryStream _stream = new MemoryStream();  
  197.   
  198.             //GZIIP处理  
  199.             if (response.ContentEncoding != null && response.ContentEncoding.Equals("gzip", StringComparison.InvariantCultureIgnoreCase))  
  200.             {  
  201.                 //开始读取流并设置编码方式  
  202.                 _stream = GetMemoryStream(new GZipStream(response.GetResponseStream(), CompressionMode.Decompress));  
  203.             }  
  204.             else  
  205.             {  
  206.                 //开始读取流并设置编码方式  
  207.                 _stream = GetMemoryStream(response.GetResponseStream());  
  208.             }  
  209.             //获取Byte  
  210.             ResponseByte = _stream.ToArray();  
  211.             _stream.Close();  
  212.             return ResponseByte;  
  213.         }  
  214.   
  215.         /// <summary>  
  216.         /// 4.0以下.net版本取数据使用  
  217.         /// </summary>  
  218.         /// <param name="streamResponse">流</param>  
  219.         private MemoryStream GetMemoryStream(Stream streamResponse)  
  220.         {  
  221.             MemoryStream _stream = new MemoryStream();  
  222.             int Length = 256;  
  223.             Byte[] buffer = new Byte[Length];  
  224.             int bytesRead = streamResponse.Read(buffer, 0, Length);  
  225.             while (bytesRead > 0)  
  226.             {  
  227.                 _stream.Write(buffer, 0, bytesRead);  
  228.                 bytesRead = streamResponse.Read(buffer, 0, Length);  
  229.             }  
  230.             return _stream;  
  231.         }  
  232.         #endregion  
  233.  
  234.         #region SetRequest  
  235.   
  236.         /// <summary>  
  237.         /// 为请求准备参数  
  238.         /// </summary>  
  239.         ///<param name="item">参数列表</param>  
  240.         private void SetRequest(HttpItem item)  
  241.         {  
  242.             // 验证证书  
  243.             SetCer(item);  
  244.             //设置Header参数  
  245.             if (item.Header != null && item.Header.Count > 0) foreach (string key in item.Header.AllKeys)  
  246.                 {  
  247.                     request.Headers.Add(key, item.Header[key]);  
  248.                 }  
  249.             // 设置代理  
  250.             SetProxy(item);  
  251.             if (item.ProtocolVersion != null) request.ProtocolVersion = item.ProtocolVersion;  
  252.             request.ServicePoint.Expect100Continue = item.Expect100Continue;  
  253.             //请求方式Get或者Post  
  254.             request.Method = item.Method;  
  255.             request.Timeout = item.Timeout;  
  256.             request.KeepAlive = item.KeepAlive;  
  257.             request.ReadWriteTimeout = item.ReadWriteTimeout;  
  258.             if (item.IfModifiedSince != null) request.IfModifiedSince = Convert.ToDateTime(item.IfModifiedSince);  
  259.             //Accept  
  260.             request.Accept = item.Accept;  
  261.             //ContentType返回类型  
  262.             request.ContentType = item.ContentType;  
  263.             //UserAgent客户端的访问类型,包括浏览器版本和操作系统信息  
  264.             request.UserAgent = item.UserAgent;  
  265.             // 编码  
  266.             encoding = item.Encoding;  
  267.             //设置安全凭证  
  268.             request.Credentials = item.ICredentials;  
  269.             //设置Cookie  
  270.             SetCookie(item);  
  271.             //来源地址  
  272.             request.Referer = item.Referer;  
  273.             //是否执行跳转功能  
  274.             request.AllowAutoRedirect = item.Allowautoredirect;  
  275.             if (item.MaximumAutomaticRedirections > 0)  
  276.             {  
  277.                 request.MaximumAutomaticRedirections = item.MaximumAutomaticRedirections;  
  278.             }  
  279.             //设置Post数据  
  280.             SetPostData(item);  
  281.             //设置最大连接  
  282.             if (item.Connectionlimit > 0) request.ServicePoint.ConnectionLimit = item.Connectionlimit;  
  283.         }  
  284.         /// <summary>  
  285.         /// 设置证书  
  286.         /// </summary>  
  287.         /// <param name="item"></param>  
  288.         private void SetCer(HttpItem item)  
  289.         {  
  290.             if (!string.IsNullOrEmpty(item.CerPath))  
  291.             {  
  292.                 //这一句一定要写在创建连接的前面。使用回调的方法进行证书验证。  
  293.                 ServicePointManager.ServerCertificateValidationCallback = new System.Net.Security.RemoteCertificateValidationCallback(CheckValidationResult);  
  294.                 //初始化对像,并设置请求的URL地址  
  295.                 request = (HttpWebRequest)WebRequest.Create(item.URL);  
  296.                 SetCerList(item);  
  297.                 //将证书添加到请求里  
  298.                 request.ClientCertificates.Add(new X509Certificate(item.CerPath));  
  299.             }  
  300.             else  
  301.             {  
  302.                 //初始化对像,并设置请求的URL地址  
  303.                 request = (HttpWebRequest)WebRequest.Create(item.URL);  
  304.                 SetCerList(item);  
  305.             }  
  306.         }  
  307.         /// <summary>  
  308.         /// 设置多个证书  
  309.         /// </summary>  
  310.         /// <param name="item"></param>  
  311.         private void SetCerList(HttpItem item)  
  312.         {  
  313.             if (item.ClentCertificates != null && item.ClentCertificates.Count > 0)  
  314.             {  
  315.                 foreach (X509Certificate c in item.ClentCertificates)  
  316.                 {  
  317.                     request.ClientCertificates.Add(c);  
  318.                 }  
  319.             }  
  320.         }  
  321.         /// <summary>  
  322.         /// 设置Cookie  
  323.         /// </summary>  
  324.         /// <param name="item">Http参数</param>  
  325.         private void SetCookie(HttpItem item)  
  326.         {  
  327.             if (!string.IsNullOrEmpty(item.Cookie)) request.Headers[HttpRequestHeader.Cookie] = item.Cookie;  
  328.             //设置CookieCollection  
  329.             if (item.ResultCookieType == ResultCookieType.CookieCollection)  
  330.             {  
  331.                 request.CookieContainer = new CookieContainer();  
  332.                 if (item.CookieCollection != null && item.CookieCollection.Count > 0)  
  333.                     request.CookieContainer.Add(item.CookieCollection);  
  334.             }  
  335.         }  
  336.         /// <summary>  
  337.         /// 设置Post数据  
  338.         /// </summary>  
  339.         /// <param name="item">Http参数</param>  
  340.         private void SetPostData(HttpItem item)  
  341.         {  
  342.             //验证在得到结果时是否有传入数据  
  343.             if (!request.Method.Trim().ToLower().Contains("get"))  
  344.             {  
  345.                 if (item.PostEncoding != null)  
  346.                 {  
  347.                     postencoding = item.PostEncoding;  
  348.                 }  
  349.                 byte[] buffer = null;  
  350.                 //写入Byte类型  
  351.                 if (item.PostDataType == PostDataType.Byte && item.PostdataByte != null && item.PostdataByte.Length > 0)  
  352.                 {  
  353.                     //验证在得到结果时是否有传入数据  
  354.                     buffer = item.PostdataByte;  
  355.                 }//写入文件  
  356.                 else if (item.PostDataType == PostDataType.FilePath && !string.IsNullOrEmpty(item.Postdata))  
  357.                 {  
  358.                     StreamReader r = new StreamReader(item.Postdata, postencoding);  
  359.                     buffer = postencoding.GetBytes(r.ReadToEnd());  
  360.                     r.Close();  
  361.                 } //写入字符串  
  362.                 else if (!string.IsNullOrEmpty(item.Postdata))  
  363.                 {  
  364.                     buffer = postencoding.GetBytes(item.Postdata);  
  365.                 }  
  366.                 if (buffer != null)  
  367.                 {  
  368.                     request.ContentLength = buffer.Length;  
  369.                     request.GetRequestStream().Write(buffer, 0, buffer.Length);  
  370.                 }  
  371.             }  
  372.         }  
  373.         /// <summary>  
  374.         /// 设置代理  
  375.         /// </summary>  
  376.         /// <param name="item">参数对象</param>  
  377.         private void SetProxy(HttpItem item)  
  378.         {  
  379.             bool isIeProxy = false;  
  380.             if (!string.IsNullOrEmpty(item.ProxyIp))  
  381.             {  
  382.                 isIeProxy = item.ProxyIp.ToLower().Contains("ieproxy");  
  383.             }  
  384.             if (!string.IsNullOrEmpty(item.ProxyIp) && !isIeProxy)  
  385.             {  
  386.                 //设置代理服务器  
  387.                 if (item.ProxyIp.Contains(":"))  
  388.                 {  
  389.                     string[] plist = item.ProxyIp.Split(‘:‘);  
  390.                     WebProxy myProxy = new WebProxy(plist[0].Trim(), Convert.ToInt32(plist[1].Trim()));  
  391.                     //建议连接  
  392.                     myProxy.Credentials = new NetworkCredential(item.ProxyUserName, item.ProxyPwd);  
  393.                     //给当前请求对象  
  394.                     request.Proxy = myProxy;  
  395.                 }  
  396.                 else  
  397.                 {  
  398.                     WebProxy myProxy = new WebProxy(item.ProxyIp, false);  
  399.                     //建议连接  
  400.                     myProxy.Credentials = new NetworkCredential(item.ProxyUserName, item.ProxyPwd);  
  401.                     //给当前请求对象  
  402.                     request.Proxy = myProxy;  
  403.                 }  
  404.             }  
  405.             else if (isIeProxy)  
  406.             {  
  407.                 //设置为IE代理  
  408.             }  
  409.             else  
  410.             {  
  411.                 request.Proxy = item.WebProxy;  
  412.             }  
  413.         }  
  414.         #endregion  
  415.  
  416.         #region private main  
  417.         /// <summary>  
  418.         /// 回调验证证书问题  
  419.         /// </summary>  
  420.         /// <param name="sender">流对象</param>  
  421.         /// <param name="certificate">证书</param>  
  422.         /// <param name="chain">X509Chain</param>  
  423.         /// <param name="errors">SslPolicyErrors</param>  
  424.         /// <returns>bool</returns>  
  425.         private bool CheckValidationResult(object sender, X509Certificate certificate, X509Chain chain, SslPolicyErrors errors) { return true; }  
  426.         #endregion  
  427.     }  
  428.     /// <summary>  
  429. 以上是关于HttpHelper工具类的主要内容,如果未能解决你的问题,请参考以下文章

    软件工程应用与实践(14)——工具类分析

    抓取百万知乎用户信息之HttpHelper的迭代之路

    csharp HttpHelper类

    csharp HttpHelper类

    HttpHelper类及调用

    博客园爬虫模拟