一个完整的大作业

Posted 2020-10-12

1.选一个自己感兴趣的主题。

www.freebuf.com   FreeBuf黑客与极客,国内关注度最高的全球互联网安全媒体平台,同时也是爱好者们交流与分享安全技术的最佳社区。

2.网络上爬取相关的数据

import requests
from bs4 import BeautifulSoup


def list(page):
    html=requests.get("http://www.freebuf.com/page/"+page)
    soup = BeautifulSoup(html.text,‘lxml‘)
    nlist=soup.select(‘div.news-info‘)
    #print(test.text)
    #print(news)
    for i in nlist:
        neirong(i.select(‘dl > dt > a‘)[0].get(‘href‘))
    try:
        list(str(int(page)+1))
    except:
        pass
    

def neirong(url):
    html=requests.get(url)
    soup = BeautifulSoup(html.text,‘lxml‘)
    t=soup.select(‘div.articlecontent‘)[0].select(‘div‘)[2].select(‘p‘)
    for i in t:
        print(i.text)
    #print(t)


list("1")

爬取数据如下

 

3.进行文本分析，生成词云

import requests
from bs4 import BeautifulSoup
import pandas as pd
import jieba

from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator
import matplotlib.pyplot as plt
from scipy.misc import imread
import os
cis= ‘‘


def list(page):
    html= requests.get("http://www.freebuf.com/page/"+ page)
    soup = BeautifulSoup(html.text, ‘lxml‘)
    nlist= soup.select(‘div.news-info‘)
    for i in nlist:
        neirong(i.select(‘dl > dt > a‘)[0].get(‘href‘))
    try:
        # list(str(int(page)+1))
        pass
    except:
        pass

def neirong(url):
    html= requests.get(url)
    soup = BeautifulSoup(html.text, ‘lxml‘)
    t= soup.select(‘div.articlecontent‘)[0].select(‘div‘)[2].select(‘p‘)
    for i in t:
        jiebafc(i.text)

def jiebafc(text):
    seg_list = jieba.cut(text, cut_all=False)
    # print ("Default Mode:", ",".join(seg_list))  # 精确模式
    ci= " " .join(seg_list)
    # print(ci)
    global cis
    cis= cis +ci


list("1")
a=[]
a.append(cis)
dataframe = pd.DataFrame({‘a_name‘:a})
dataframe.to_csv("test.csv",index =False,sep= ‘ ‘)
print(cis)

d =os.path.dirname(__file__)  # 获取当前路径
text= cis
alice_coloring = imread(os.path.join(d, "11.png"))  # 设置背景图片

# 初始化词云一系列参数
wc = WordCloud(background_color="white",  # 背景颜色max_words=2000,# 词云显示的最大词数
               mask=alice_coloring,  # 设置背景图片
               stopwords=STOPWORDS.add("said"),
               font_path="MSYH.TTF",  # 中文字体文件
               max_font_size=40,  # 字体最大值
               random_state=42)
wc.generate(text)  # 生成词云
image_colors = ImageColorGenerator(alice_coloring)  # 从背景图片生成颜色值

plt.imshow(wc)  # 画词云
plt.axis("off")
wc.to_file(os.path.join(d, "名称.png"))  # 保存图片到当前目录下

截图如下