爬虫_斗图啦(队列，多线程)

Posted 2021-01-02 mc-curry

 1 import threading
 2 import requests
 3 from lxml import etree
 4 from urllib import request
 5 import os
 6 import re
 7 from queue import Queue
 8 
 9 
10 class Producer(threading.Thread):
11     headers = {
12         ‘User-Agent‘: ‘Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (Khtml, like Gecko) Chrome/62.0.3202.94 Safari/537.36‘
13     }
14     def __init__(self,page_queue,img_queue,*args,**kwargs):
15         super(Producer, self).__init__(*args,**kwargs)
16         self.page_queue = page_queue
17         self.img_queue = img_queue
18 
19 
20     def run(self):
21         while True:
22             if self.page_queue.empty():
23                 break
24             url = self.page_queue.get()
25             self.parse_page(url)
26 
27 
28     def parse_page(self,url):
29         response = requests.get(url,headers=self.headers)
30         text = response.text
31         html = etree.HTML(text)
32         imgs = html.xpath("//div[@class=‘page-content text-center‘]//a//img")
33         for img in imgs:
34             if img.get(‘class‘) == ‘gif‘:
35                 continue
36             img_url = img.xpath(".//@data-original")[0]
37             suffix = os.path.splitext(img_url)[1]
38             alt = img.xpath(".//@alt")[0]
39             alt = re.sub(r‘[，。？?,/\·]‘,‘‘,alt)
40             img_name = alt + suffix
41             self.img_queue.put((img_url,img_name))
42 
43 
44 class Consumer(threading.Thread):
45     def __init__(self,page_queue,img_queue,*args,**kwargs):
46         super(Consumer, self).__init__(*args,**kwargs)
47         self.page_queue = page_queue
48         self.img_queue = img_queue
49 
50 
51     def run(self):
52         while True:
53             if self.img_queue.empty():
54                 if self.page_queue.empty():
55                     return
56             img = self.img_queue.get(block=True)
57             url,filename = img
58             request.urlretrieve(url,‘images/‘+filename)
59             print(filename+‘  下载‘)
60 
61 
62 def main():
63     page_queue = Queue(100)
64     img_queue = Queue(500)
65 
66     for x in range(1,101):
67         url = "http://www.doutula.com/photo/list/?page=%d" % x
68         page_queue.put(url)
69     for x in range(5):
70         t = Producer(page_queue,img_queue)
71         t.start()
72     for x in range(5):
73         t = Consumer(page_queue,img_queue)
74         t.start()
75 
76 
77 if __name__ == ‘__main__‘:
78     main()

下载是相当快啊