python—多协程爬取斗鱼高颜值美女图片

Posted 2021-02-08 Summer儿

斗鱼已将网页的源代码修改，若想爬取图片请转至
https://www.cnblogs.com/summer1019/p/10388348.html
 1 import requests
 2 from bs4 import BeautifulSoup
 3 from urllib import request
 4 # import threading
 5 import gevent
 6 from gevent import monkey
 7 
 8 monkey.patch_all()
 9 
10 def get_html_text(url):
11     try:
12         r = requests.get(url, timeout=10)
13         r.raise_for_status()
14         r.encoding = r.apparent_encoding
15         return r.text
16     except Exception as result:
17         print(\'错误类型：\', result)
18 
19 
20 def html_text_parser(img_list, html):
21     soup = BeautifulSoup(html, \'html.parser\')
22     for link in soup.find_all(\'img\'):
23         Link = link.get(\'data-original\')  #link.get(\'key\')--->>获取属性值
24         if Link:
25             img_list.append(Link)
26     return img_list
27 
28 
29 def get_douyu_img(Img_list):
30     for i,j in enumerate(Img_list):
31         # name = j.split(\'.\')[-1]
32         r = request.urlopen(j)
33         ima_content = r.read()
34         path = str(i)
35         with open(path, \'wb\') as f:
36             f.write(ima_content)
37 
38 def main():
39     url = \'https://www.douyu.com/g_yz\'
40     html = get_html_text(url)
41     img_list = list()
42     Img_list = html_text_parser(img_list, html)
43     # print(Img_list)
44     #t1 = threading.Thread(target=get_html_text, args=(url,))
45     #t2 = threading.Thread(target=html_text_parser, args=(img_list,html))
46     #t3 = threading.Thread(target=get_douyu_img, args=(Img_list,))
47     #t1.start()
48     #t2.start()
49     #t3.start()
50     gevent.joinall([
51         gevent.spawn(get_html_text, url),
52         gevent.spawn(html_text_parser, img_list, html),
53         gevent.spawn(get_douyu_img, Img_list)
54     ])
55 
56 
57 if __name__ == \'__main__\':
58     main()

备注:
一次性可以爬取239张高颜值美女图片，不同时段爬取的主播不同~
权当娱乐，分享诸位，
完全自学自练，欢迎指正。