Part of Speech Tagging

Natural Language Processing with Python

Charpter 6.1

suffix_fdist处代码稍微改动。

import nltk
from nltk.corpus import brown

def common_suffixes_fun():
    suffix_fdist=nltk.FreqDist()
    for word in brown.words():
        word=word.lower()
        suffix_fdist[word[-1:]] +=1
        suffix_fdist[word[-2:]] +=1
        suffix_fdist[word[-3:]] +=1
    most_freqent_items=[it for it in sorted(suffix_fdist.items(),key=lambda x:(-x[1],x[0]))[:100]]
    return [su[0] for su in most_freqent_items]

common_suffixes = common_suffixes_fun()

def pos_features(word):
    features={}
    for su in common_suffixes:
        features['endswith(%s)' % su]=word.lower().endswith(su)
    return features
    
def test_pos():
    tagged_words = brown.tagged_words(categories='news')[:5000]
    featuresets=[(pos_features(word),tag) for (word,tag) in tagged_words]
    
    size= int(len(tagged_words)*0.1)
    train_set, test_set = featuresets[size:],featuresets[:size]
    classifier=nltk.NaiveBayesClassifier.train(train_set)
    
    print nltk.classify.accuracy(classifier,test_set)
    classifier.show_most_informative_features(5)

运行结果为：

0.652
Most Informative Features
endswith(o) = True TO : NN = 423.2 : 1.0
endswith(es) = True DOZ : NN = 319.5 : 1.0
endswith(om) = True WPO : NN = 319.5 : 1.0
endswith(as) = True BEDZ : IN = 303.3 : 1.0
endswith(s) = True BEDZ : IN = 303.3 : 1.0