NLP整体流程的代码

import nltk
import numpy as np
import re
from nltk.corpus import stopwords

# 1 分词1
text = "Sentiment analysis is a challenging subject in machine learning.
 People express their emotions in language that is often obscured by sarcasm,
  ambiguity, and plays on words, all of which could be very misleading for 
  both humans and computers. There's another Kaggle competition for movie review 
  sentiment analysis. In this tutorial we explore how Word2Vec can be applied to 
  a similar problem.".lower()

text_list = nltk.word_tokenize(text)

#2 q去掉标点符号和停用词
#去掉标点符号
english_punctuations = [',', '.', ':', ';', '?', '(', ')', '[', ']', '&', '!', '*', '@', '#', '$', '%']
text_list = [word for word in text_list if word not in english_punctuations]
#去掉停用词
stops = set(stopwords.words("english"))
text_list = [word for word in text_list if word not in stops]

#3统计词频
freq_dist = nltk.FreqDist(text_list)
freq_list = []
num_words = len(freq_dist.values())
for i in range(num_words):
    freq_list.append([list(freq_dist.keys())[i],list(freq_dist.values())[i]])
freqArr = np.array(freq_list)
print(freqArr)

#4词性标注
print(nltk.pos_tag(text_list))