爬虫学习——带cookie的网页进行爬取

Posted 2021-02-09 kuangkuangduangduang

# 前提：
# 
# 通常，很多网站需要登录才能进行浏览，所以在爬取这些网站时，也需要进行登录，并拿取登录时的cookie
# 
# 登录网页，服务器会给客户端一个牌子cookie
# 
# 访问登录页面时，带着牌子进行请求才能返回响应
# 


# 登录界面的爬取

# 做法：

    # 找到牌子，带着牌子进行请求

    # cookie有的在请求头里

# 如下是在登录后的页面中找到请求头里的cookie，然后进行请求，访问其含登陆信息的页面

import urllib.request
import urllib.parse

#将带cookie请求头信息添加到请求对象中取
headers = {
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
    # "Accept-Encoding": "gzip, deflate",
    # "Accept-Language": "zh-CN,zh;q=0.9",
    "Connection": "keep-alive",
    "Cookie": "anonymid=js2wkb2xx3aylq; depovince=GW; _r01_=1; JSESSIONID=abcwlggMRpipBajTf3LJw; ick_login=7c1fa03b-b8cf-408e-998a-6f7d34abd0d7; t=49a13f402543c813e2c6d684147af8133; societyguester=49a13f402543c813e2c6d684147af8133; id=969726303; xnsid=1a234058; jebecookies=bc2283c3-38f0-468f-b754-8f6550e1b52a|||||; ver=7.0; loginfrom=null; springskin=set; jebe_key=6b5b8da6-ae2c-4d26-ab60-66bb55a70491%7C1b833888a1eb6aca75ec4170a8e04c2d%7C1550044677566%7C1%7C1550044653252; vip=1; ch_id=10013; _ga=GA1.2.222558484.1550044669; _gid=GA1.2.491107391.1550044669; wp_fold=0",
    "Host": "www.renren.com",
    "Referer": "http://www.renren.com/969726303/profile",
    "Upgrade-Insecure-Requests": "1",
    "User-Agent":" Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36",
}

#需要登录验证的网页网址
url = "http://www.renren.com/969726303/profile?v=info_timeline"
request = urllib.request.Request(url,headers=headers)
response = urllib.request.urlopen(request)
with open("renren.html","wb")as tf:
    tf.write(response.read())
    tf.close()


# 如下是在登录时就直接抓取登录时服务器给的cookie数据，然后在之后访问其他需要登录验证的网页时带着cookie进行访问就行了