获取URL网页信息

Posted 2020-09-19 倪子

 

 

        static string Gethtml(string url)
        {string strHTML = "";
            WebClient myWebClient = new WebClient();
            Stream myStream = myWebClient.OpenRead(url);
            StreamReader sr = new StreamReader(myStream, System.Text.Encoding.GetEncoding("utf-8"));
            strHTML = sr.ReadToEnd();
            myStream.Close();
            return strHTML;
        }


        static bool GetHtml(string url)
      {
    
            HttpWebRequest myHttpWebRequest = (HttpWebRequest)WebRequest.Create(url);
            myHttpWebRequest.Timeout = 20 * 1000; //连接超时
            myHttpWebRequest.Accept = "*/*";
            myHttpWebRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0;)";
            HttpWebResponse myHttpWebResponse = (HttpWebResponse)myHttpWebRequest.GetResponse();
            Stream stream = myHttpWebResponse.GetResponseStream();
            StreamReader sr =
               new StreamReader(stream, Encoding.GetEncoding("utf-8"));
            var html = sr.ReadToEnd();
           
        }

 

 

获取当前Cookies保存后获取整站信息

static void Main(string[] args)
        {
           

            Console.WriteLine("确保挂载VPN后。。。回车键继续！");
            Console.ReadKey();

            string url = "http://www.sciencedirect.com/science/journal/00118486/61/6";

            string indata = "aa=zhuye";
            CookieContainer myCookieContainer =
                new CookieContainer();
            //新建一个cookiecontainer来存放cookie集合
            HttpWebRequest myHttpWebRequest =
                (HttpWebRequest)WebRequest.Create(url);
            //新建一个httpwebrequest
            myHttpWebRequest.ContentType = "application/x-www-form-urlencoded";
            myHttpWebRequest.ContentLength = indata.Length;
            myHttpWebRequest.Method = "post";
            myHttpWebRequest.CookieContainer = myCookieContainer;
            //设置httpwebrequest的cookiecontainer为
            //刚才建立的那个mycookiecontainer
            Stream myRequestStream = myHttpWebRequest.GetRequestStream();
            StreamWriter myStreamWriter = new StreamWriter(myRequestStream, Encoding.GetEncoding("gb2312"));
            myStreamWriter.Write(indata);
            //把数据写入httpwebrequest的request流
            myStreamWriter.Close();
            myRequestStream.Close();
            //关闭打开对象
            HttpWebResponse myHttpWebResponse =
                (HttpWebResponse)myHttpWebRequest.GetResponse();
            //新建一个httpwebresponse
            myHttpWebResponse.Cookies = myCookieContainer.GetCookies(myHttpWebRequest.RequestUri);
          
            Console.WriteLine("获取cookies成功！。。。关闭VPN后任意键继续");
            Console.ReadKey();

            var htmlData = SaveCook(myHttpWebRequest, prevUrl, myCookieContainer, myHttpWebResponse);
                               
            }
        }

 

 public static string SaveCook(HttpWebRequest myHttpWebRequest, string url, CookieContainer myCookieContainer, HttpWebResponse myHttpWebResponse)
        {
            //拿到了cookie，再进行请求就能直接读取到登录后的内容了
            myHttpWebRequest = (HttpWebRequest)WebRequest.Create(url);
            myHttpWebRequest.CookieContainer = myCookieContainer;//*
                                                                 //刚才那个cookiecontainer已经存有了cookie,把它附加到
 
            myHttpWebResponse = (HttpWebResponse)myHttpWebRequest.GetResponse();
            myHttpWebResponse.Cookies =
                myCookieContainer.GetCookies(myHttpWebRequest.RequestUri);


            Stream myresponsestream = myHttpWebResponse.GetResponseStream();
            StreamReader mystreamreader =
               new StreamReader(myresponsestream, Encoding.GetEncoding("gb2312"));

            var html = mystreamreader.ReadToEnd();
            //把数据从httpwebresponse的response流中读出
            mystreamreader.Close();
            myresponsestream.Close();
            return html;

        }