去停用词

Posted 2022-11-17 rise0111

import pandas as pd
stop_words = []
with open('data/stop_words.txt','r',encoding='utf-8') as f:
    lines = f.readlines()
    for i in lines:
        word = i.strip()
        stop_words.append(word)
print(stop_words[:10])


def clean_stopwords(text,stop_words):
    data = []
    for sentence in text:
        data.append([ word for word in sentence if word not in stop_words])
    return data