利用bs4爬取三国演义所有章节标题以及章节内容

Posted 2020-11-15 jnhnsnow

　　url = ‘ http://www.shicimingju.com/book/sanguoyanyi.html‘

　　

from bs4 import BeautifulSoup
import requests
url = ‘http://www.shicimingju.com/book/sanguoyanyi.html‘
headers = {
‘User-Agent‘: ‘Mozilla/5.0 (Linux; android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Mobile Safari/537.36‘
}

page_text = requests.get(url=url,headers=headers).text
soup = BeautifulSoup(page_text,‘lxml‘)
res_list = soup.select(‘.book-mulu a‘)
with open(‘三国演义.text‘,‘w‘,encoding=‘utf-8‘)as f:
    for item in res_list:
        url_item = ‘%s%s‘%("http://www.shicimingju.com",item[‘href‘])
        detail_page_text = requests.get(url=url_item, headers=headers).text
        detail_soup =  BeautifulSoup(detail_page_text,‘lxml‘)
        title = detail_soup.find(‘div‘,class_=‘www-main-container‘).text
        body = detail_soup.find("div",class_=‘chapter_content‘).text
        f.write(title+‘
‘+body)