Python爬虫之利用BeautifulSoup爬取豆瓣小说——将小说信息写入文件

Posted 2020-10-05

 1 #-*-coding:utf-8-*-
 2 import urllib2
 3 from bs4 import BeautifulSoup
 4 
 5 class dbxs:
 6 
 7     def __init__(self):
 8         self.pageIndex = 0
 9         self.enable = True
10         self.file = None
11         self.content = []
12         
13         
14     #获取html页面的内容
15     def getPage(self, pageIndex):
16         try:
17             #设置代理ip
18             enable_proxy = True
19             proxy_handler = urllib2.ProxyHandler({‘Http‘: ‘113.118.170.230:808‘})
20             null_proxy_handler = urllib2.ProxyHandler({})
21             if enable_proxy:
22                 opener = urllib2.build_opener(proxy_handler)
23             else:
24                 opener = urllib2.build_opener(null_proxy_handler)
25             urllib2.install_opener(opener)
26             #获得页面响应的内容
27             url = ‘https://www.douban.com/tag/%E5%B0%8F%E8%AF%B4/book‘ + "?start=" + str(pageIndex)
28             #设置请求头部信息，模拟浏览器的行为
29             my_headers = {‘User-Agent‘:‘Mozilla/5.0 (Windows NT 6.1; WOW64; rv:55.0)‘}
30             request =   urllib2.Request(url, headers = my_headers)
31             response = urllib2.urlopen(request)
32             return response.read()
33         except urllib2.URLError, e:
34             if hasattr(e, "code"):
35                 print e.code
36             if hasattr(e, "reason"):
37                 print e.reason
38                 return None
39 
40     #过滤查找这一页的小说名字，信息和评分
41     def getContent(self, pageIndex, content):
42         pageCode = self.getPage(pageIndex)
43         soup = BeautifulSoup(pageCode, ‘html.parser‘)
44         #在获得相应的内容中找出所有标签为<dd>的内容（里面包含了我们需要的小说信息）
45         contents = soup.find_all(‘dd‘)
46         
47         if contents:
48             for item in contents:
49                 title = item.find(class_ = ‘title‘).string.encode(‘utf-8‘)
50                 info = item.find(class_ = ‘desc‘).string.strip().encode(‘utf-8‘)
51                 rate = item.find(class_ = ‘rating_nums‘)
52                 #通过试验，我们发现某一页可能存在小说没有评分，如果我们不判断rate，那么可能就出现报错
53                 if rate:
54                     rates = rate.string.encode(‘utf-8‘)
55                     content.append([title, info, rates])
56                     
57                 else:
58                     content.append([title, info])
59         #如果页面不包含<dd>标签，我们应该停止
60         else:
61             print u"所有页面已加载完"
62             self.enable = False
63 
64         return content
65         
66             
67 
68     #写入文件
69     def writeData(self, content):
70         self.file = open("bdxs.txt", "w+")   #必须在for循环外面，不然每一次写入都会覆盖之前的数据
71         for item in content:
72             if len(item) == 3:
73                 self.file.write(item[0] + "\n")
74                 self.file.write(item[1] + "\n")
75                 self.file.write(u"评分:" + item[2] + "\n\n")
76             else:
77                 self.file.write(item[0] + "\n")
78                 self.file.write(item[1] + "\n")
79             self.file.write("========================================\n\n")
80 
81             
82     #创建一个开始方法
83     def start(self):
84         x = 1
85         while self.enable == True:
86             content = self.getContent(self.pageIndex, self.content)
87             if self.enable == True:
88                 print "正在写入第%s页..." %x
89             self.writeData(content)
90             self.pageIndex += 15
91             x += 1
92                       
93             
94 DBXS = dbxs()
95 DBXS.start()

这段代码我还没理解透彻，比如每一页的小说信息写入完成后，怎么在后面加上第几页，后期我将继续完善它。