爬虫基本功系列之 -- requests库的使用

Posted 2021-07-18 一腔诗意醉了酒

文章目录

• 1、安装包
• 2、 基本使用
• 3、下载文件

---

1、安装包

python pip3 install requests

2、 基本使用

# 构建复杂请求

import requests

# 按需构建headers

url = "https://www.baidu.com"

headers = {
    "Content-type" : "application/json",
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (Khtml, like Gecko) Chrome/90.0.4430.93 Safari/537.36 Edg/90.0.818.56"
}

proxies = {
    "http":"http://192.168.204.1:3000",
    "https":"http://192.168.204.1:8080"
}
# 加上headers请求
# r = requests.get(url,headers=headers)


# 使用代理ip
# r = requests.get(url, proxies=proxies)


# 关闭证书验证  verify=True/False/证书路径字符串
# r = requests.get(url,verify=False)

# 设置超时
# try:
#     r = requests.get(url, timeout=0.1)
#     print(r.content.decode('utf-8'))

# except:
#     print("出现异常")


# 按需使用cookie

temp_cookies = "name=qq;pwd=123"

cookie_dict = {}
for i in temp_cookies.split(';'):
    value = i.split('=')
    cookie_dict[value[0]] = value[1]

print(cookie_dict)

# r = requests.get(url=url, cookies=cookie_dict)

# 获取响应的cookie

# 可以混合使用
r = requests.get(url, headers=headers, cookies=cookie_dict)

print(r.cookies)
print(r.status_code)
print(r.content.decode('utf-8'))

3、下载文件

import requests;

url = 'http://img.aiimg.com/uploads/userup/0909/2Z64022L38.jpg'

r= requests.get(url)
if(r.status_code==200):
    with open('python.png','wb+') as f:
        f.write(r.content)
        print("下载完毕")

---

合法使用爬虫

---