LDA主题模型

Posted 2021-01-23 lifengwu

#!/usr/bin/env python
# -*- coding:utf-8 -*-
import numpy as np
import pandas as pd
import re
df = pd.read_csv("HillaryEmails.csv")
df = df[[‘Id‘,‘ExtractedBodyText‘]].dropna()#保留这两个信息，其他的扔掉
#文本预处理
def clean_email_text(text):
    text = text.replace(‘/n‘," ")#去掉新行
    text = re.sub(r‘-‘,‘ ‘,text)
    text = re.sub(r"d+/d+/d+","",text)
    text = re.sub(r"[0-2]?[0-9]:[0-6][0-9]","",text)
    text = re.sub(r"[w][email protected][.w]+","",text)
    text = re.sub(r"/[a-zA-Z]*[://]*[A-Za-z0-9-_]+.+[A-Za-z0-9]./"
                  r"%&=?-_]+/i","",text)
    pure_text = ‘‘
    for letter in text:
        if letter.isalpha() or letter==‘ ‘: #只留下字母和空格
            pure_text +=letter
    #去除落单的单词
    text = ‘ ‘.join(word for word in pure_text.split() if len(word)>1)
    return text
#新建一个colum，把方法跑一遍
docs = df[‘ExtractedBodyText‘]
docs = docs.apply(lambda s: clean_email_text(s))
print(docs.head(1).values)
doclist = docs.values
#引入库
from gensim import corpora,models,similarities
import gensim
stoplist = [‘very‘, ‘ourselves‘, ‘am‘, ‘doesn‘, ‘through‘, ‘me‘, ‘against‘, ‘up‘, ‘just‘, ‘her‘, ‘ours‘,
            ‘couldn‘, ‘because‘, ‘is‘, ‘isn‘, ‘it‘, ‘only‘, ‘in‘, ‘such‘, ‘too‘, ‘mustn‘, ‘under‘, ‘their‘,
            ‘if‘, ‘to‘, ‘my‘, ‘himself‘, ‘after‘, ‘why‘, ‘while‘, ‘can‘, ‘each‘, ‘itself‘, ‘his‘, ‘all‘, ‘once‘,
            ‘herself‘, ‘more‘, ‘our‘, ‘they‘, ‘hasn‘, ‘on‘, ‘ma‘, ‘them‘, ‘its‘, ‘where‘, ‘did‘, ‘ll‘, ‘you‘,
            ‘didn‘, ‘nor‘, ‘as‘, ‘now‘, ‘before‘, ‘those‘, ‘yours‘, ‘from‘, ‘who‘, ‘was‘, ‘m‘, ‘been‘, ‘will‘,
            ‘into‘, ‘same‘, ‘how‘, ‘some‘, ‘of‘, ‘out‘, ‘with‘, ‘s‘, ‘being‘, ‘t‘, ‘mightn‘, ‘she‘, ‘again‘, ‘be‘,
            ‘by‘, ‘shan‘, ‘have‘, ‘yourselves‘, ‘needn‘, ‘and‘, ‘are‘, ‘o‘, ‘these‘, ‘further‘, ‘most‘, ‘yourself‘,
            ‘having‘, ‘aren‘, ‘here‘, ‘he‘, ‘were‘, ‘but‘, ‘this‘, ‘myself‘, ‘own‘, ‘we‘, ‘so‘, ‘i‘, ‘does‘, ‘both‘,
            ‘when‘, ‘between‘, ‘d‘, ‘had‘, ‘the‘, ‘y‘, ‘has‘, ‘down‘, ‘off‘, ‘than‘, ‘haven‘, ‘whom‘, ‘wouldn‘,
            ‘should‘, ‘ve‘, ‘over‘, ‘themselves‘, ‘few‘, ‘then‘, ‘hadn‘, ‘what‘, ‘until‘, ‘won‘, ‘no‘, ‘about‘,
            ‘any‘, ‘that‘, ‘for‘, ‘shouldn‘, ‘don‘, ‘do‘, ‘there‘, ‘doing‘, ‘an‘, ‘or‘, ‘ain‘, ‘hers‘, ‘wasn‘,
            ‘weren‘, ‘above‘, ‘a‘, ‘at‘, ‘your‘, ‘theirs‘, ‘below‘, ‘other‘, ‘not‘, ‘re‘, ‘him‘, ‘during‘, ‘which‘]
texts = [[word for word in doc.lower().split() if word not in stoplist] for doc in doclist]
print(texts[0])
#建立语料库
dictionary = corpora.Dictionary(texts)
corpus = [dictionary.doc2bow(text) for text in texts]
print(corpus[13])
#建立模型
lda = gensim.models.ldamodel.LdaModel(corpus=corpus,id2word=dictionary,num_topics=20)
print(lda.print_topic(10,topn=5))
print(lda.print_topics(num_topics = 10,num_words = 5))
lda_list = []  #doc1这句话属于哪个主题？
doc1 = ‘To all the little girls watching never doubt that you are valuable and powerful & deserving of every chance & opportunity in the world‘
for words in doc1:
    doc_bow = dictionary.doc2bow(words)
    doc_lda = lda[doc_bow]
lda_list.append(doc_lda)
print(lda_list)