python应用：主题分类（gensim lda）

安装第三方包：gensim

首先，执行去停词操作（去除与主题无关的词）

#-*-coding:utf8-*-

import jieba

def stopwordslist(filepath):
    stopwords = [line.strip() for line in open(filepath, 'r').readlines()]
    return stopwords

def seg_sentence(sentence):
    sentence_seged = jieba.cut(sentence.strip())
    stopwords = stopwordslist('stopWords/stopwords.txt')
    outstr = ''
    for word in sentence_seged:
        word = word.lower()
        if word not in stopwords:
            if word != '	':
                outstr += word
                outstr += " "
    return outstr

inputs = open('input/copurs.txt', 'r')   

outputs = open('input/copurs_out.txt', 'w')
for line in inputs:
    line_seg = seg_sentence(line)
    outputs.write(line_seg + '
')
outputs.close()
inputs.close()

然后，执行主题分类操作

import codecs
from gensim import corpora
from gensim.models import LdaModel
from gensim import models
from gensim.corpora import Dictionary


te = []
fp = codecs.open('input/copurs_out.txt','r')
for line in fp:
    line = line.split()
    te.append([ w for w in line  ])
print len(te) 
dictionary = corpora.Dictionary(te)
corpus = [ dictionary.doc2bow(text) for text in te ]

#tfidf = models.TfidfModel(corpus)
#corpus_tfidf = tfidf[corpus]

#########Run the LDA model for XX topics ###############################
lda = LdaModel(corpus=corpus, id2word=dictionary, num_topics=50,passes=2000) 
doc_topic = [a for a in lda[corpus]]

####### write the topics in file topics_result.txt ####################
topics_r = lda.print_topics(num_topics = 50, num_words = 10)
topic_name = codecs.open('output/topics_result.txt','w')
for v in topics_r:
    topic_name.write(str(v)+'
')

    
######################  write the class results to file  #########################
###################### each document belongs to which topic ######################

fp2 = codecs.open('output/documents_result.txt','w')
for t in doc_topic:
    c = []
    c.append([a[1] for a in t])
    m = max(c[0])
    
    for i in range(0, len(t)):
        if m in t[i]:
            #print(t[i])
            fp2.write(str(t[i][0]) + '  ' + str(t[i][1]) + '
')
            break
################################ OVER ############################################

注意：上述主题分类，仅使用lda模型（根据频数计算）

也可混合使用tf-idf模型XX-topic下代码改为如下即可：

方式一
#########Run the LDA model for XX topics ###############################
lda = LdaModel(corpus=corpus_tfidf, id2word=dictionary, num_topics=50,passes=2000) 
doc_topic = [a for a in lda[corpus_tfidf]]

或

方式二
#########Run the LDA model for XX topics ###############################
lda = LdaModel(corpus=corpus, id2word=dictionary, num_topics=50,passes=2000) 
doc_topic = [a for a in lda[corpus_tfidf]]

常用方式为方式一，作者暂时为弄清楚这两种方式的区别，后期将会继续完善