爬虫的浏览器伪装技术（高度伪装）

Posted 2020-09-27 小明在线

 1 import urllib.request
 2 import http.cookiejar
 3 
 4 url = "http://www.baidu.com"
 5 file_path = "E:/workspace/PyCharm/codeSpace/books/python_web_crawler_book/chapter6/demo5/1.html"
 6 
 7 # 添加报头 注意"Accept-Encoding": "gb2312, utf-8" 防止解码而出现乱码
 8 headers = {"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8","Accept-Encoding": "gb2312, utf-8","Accept-Language": "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3","User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0",    "Connection": "keep-alive","Host": "baidu.com"
 9 }
10 cjar = http.cookiejar.CookieJar()
11 opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cjar))
12 headall = []
13 for key,value in headers.items():
14     item = (key,value)
15     headall.append(item)
16 opener.addheaders = headall
17 urllib.request.install_opener(opener)
18 print(urllib.request.urlopen(url).read().decode(‘utf-8‘))