requests的封装(user-agent,proxies)

Posted 2020-10-20 yugengde

import requests

#  1. headers

url = ‘https://api.github.com/some/endpoint‘
headers = {‘user-agent‘: ‘Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (Khtml, like Gecko) Chrome/62.0.3202.89 Safari/537.36‘}

r = requests.get(url, headers=headers)

#  2. proxies

proxies = {
  ‘http‘: ‘http://10.10.1.10:3128‘,
  ‘https‘: ‘http://10.10.1.10:1080‘,
}

requests.get(‘http://example.org‘, proxies=proxies)


@example version_1.0 封装 requests

def proxy_request(url):
   def get_proxy(retries=5):
      if retries:
         proxy = requests.get(‘http://localhost:5000/proxy/get‘)
         if proxy.status_code == 200:
            return proxy.text
         else:
            get_proxy(retries - 1)
      else:
         return None

   proxy = get_proxy()
   if proxy is not None:
      proxies = {"http": "http://%s" % proxy}
   else:
      print(‘***********************代理池没有IP可用啦!!!**********************‘)
      time.sleep(15)
      proxies = {"http": "http://%s" % "127.0.0.0:1080"}

   headers = {‘user-agent‘:UserAgent().random}
   r = requests.get(url=url,headers=headers,proxies=proxies)
   return r