爬虫大作业

Posted 2020-11-02 宇健

import requests, re, jieba,pandas
from bs4 import BeautifulSoup
from datetime import datetime
from wordcloud import WordCloud
import matplotlib.pyplot as plt

# 获取新闻细节
def getNewsDetail(newsUrl):
    res = requests.get(newsUrl)
    res.encoding = \'gb2312\'
    soupd = BeautifulSoup(res.text, \'html.parser\')
    detail = {\'title\': soupd.select(\'#epContentLeft\')[0].h1.text, \'newsUrl\': newsUrl, \'time\': datetime.strptime(
        re.search(\'(\\d{4}.\\d{2}.\\d{2}\\s\\d{2}.\\d{2}.\\d{2})\', soupd.select(\'.post_time_source\')[0].text).group(1),
        \'%Y-%m-%d %H:%M:%S\'), \'source\': re.search(\'来源:(.*)\', soupd.select(\'.post_time_source\')[0].text).group(1),
              \'content\': soupd.select(\'#endText\')[0].text}
    return detail

# 通过jieba分词，获取新闻关键词
def getKeyWords():
    content = open(\'news.txt\', \'r\', encoding=\'utf-8\').read()
    wordSet = set(jieba._lcut(\'\'.join(re.findall(\'[\\u4e00-\\u9fa5]\', content))))  # 通过正则表达式选取中文字符数组，拼接为无标点字符内容,再转换为字符集合
    wordDict = {}
    deleteList, keyWords = [], []
    for i in wordSet:
        wordDict[i] = content.count(i)  # 生成词云字典
    for i in wordDict.keys():
        if len(i) < 2:
            deleteList.append(i)  # 生成单字无意义字符列表
    for i in deleteList:
        del wordDict[i]  # 在词云字典中删除无意义字符
    dictList = list(wordDict.items())
    dictList.sort(key=lambda item: item[1], reverse=True)
    for dict in dictList:
        keyWords.append(dict[0])
    writekeyword(keyWords)


# 将新闻内容写入到文件
def writeNews(pagedetail):
    f = open(\'text1.txt\', \'a\', encoding=\'utf-8\')
    for detail in pagedetail:
        f.write(detail[\'content\'])
    f.close()


# 将词云写入到文件
def writekeyword(keywords):
    f = open(\'text.txt\', \'a\', encoding=\'utf-8\')
    for word in text:
        f.write(\'  \' + word)
    f.close()

# 获取一页的新闻
def getListPage(listUrl):
    res = requests.get(listUrl)
    res.encoding = \'utf-8\'
    soup = BeautifulSoup(res.text, \'html.parser\')
    pagedetail = []  # 存储一页所有新闻的详情
    for news in soup.select(\'#news-flow-content\')[0].select(\'li\'):
        newsdetail = getNewsDetail(news.select(\'a\')[0][\'href\'])  # 调用getNewsDetail()获取新闻详情
        pagedetail.append(newsdetail)
    return pagedetail


def getWordCloud():
    keywords = open(\'keywords.txt\', \'r\', encoding=\'utf-8\').read()  # 打开词云文件
    wc = WordCloud(font_path=r\'C:\\Windows\\Fonts\\simfang.ttf\', background_color=\'white\', max_words=100).generate(
        keywords).to_file(\'kwords.png\')  # 生成词云，字体设置为可识别中文字符
    plt.imshow(wc)
    plt.axis(\'off\')
    plt.show()

pagedetail = getListPage(\'http://tech.163.com/internet/\')  # 获取首页新闻
writeNews(pagedetail)
for i in range(2, 20):  # 因为网易新闻频道只存取20页新闻，直接设置20
    listUrl = \'http://tech.163.com/special/tele_2016_%02d/\' % i  # 填充新闻页，页面格式为两位数字字符
    pagedetail = getListPage(listUrl)
    writeNews(pagedetail)
getKeyWords()  # 获取词云，并且写到文件
getWordCloud()  # 从词云文件读取词云，生成词云