python反爬之用户代理

Posted 2021-01-21 嘎嘣real

# requests是第三方库，需要安装 pip install requests
import requests
import random
# 通常很多网站都会设置检测请求头中的User-Agent，所以在编写爬虫代码时一般都会加上user-agent
url = ‘http://www.zhihu.com‘

# 如果同一个user-agent请求次数过多，可能也可能被检测出来,所以我们可以写一个列表，里面存放很多的user-agent，每次请求在列表中随机抽取一个
user_list = ["Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/534.7 (KHTML, like Gecko) Chrome/7.0.514.0 Safari/534.7",
    "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/534.14 (KHTML, like Gecko) Chrome/9.0.601.0 Safari/534.14",
    "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.14 (KHTML, like Gecko) Chrome/10.0.601.0 Safari/534.14",
    "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.20 (KHTML, like Gecko) Chrome/11.0.672.2 Safari/534.20",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.27 (KHTML, like Gecko) Chrome/12.0.712.0 Safari/534.27",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.24 Safari/535.1",
    "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.120 Safari/535.2",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.36 Safari/535.7",
    "Mozilla/5.0 (Windows; U; Windows NT 6.0 x64; en-US; rv:1.9pre) Gecko/2008072421 Minefield/3.0.2pre",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.10) Gecko/2009042316 Firefox/3.0.10",
    "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-GB; rv:1.9.0.11) Gecko/2009060215 Firefox/3.0.11 (.NET CLR 3.5.30729)",
    "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 GTB5",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; tr; rv:1.9.2.8) Gecko/20100722 Firefox/3.6.8 ( .NET CLR 3.5.30729; .NET4.0E)",
    "Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1"]

headers = {
    ‘User-Agent‘:random.choice(user_list)
}
# 可以打印查看每次选出了哪一个user-agent
print(headers)

r = requests.get(url=url,headers=headers)
# 也可以试一下，如果不加headers会不会返回正确的状态码
print(r.status_code)
‘‘‘
常见的状态码及含义：
    200 OK     一切正常
    301 Moved Permanently     重定向到新的URL，永久性
    302 Found     重定向到临时的URL，非永久性
    304 Not Modified     请求的资源未更新
    400 Bad Request     非法请求
    401 Unauthorized     请求未经授权
    403 Forbidden     禁止访问
    404 Not Found     没有找到对应页面
    500 Internal Server Error     服务器内部出现错误
    501 Not Implemented     服务器不支持实现请求所需要的功能
‘‘‘