Sequence Classification

Natural Language Processing with Python

Charpter 6.1

import nltk
from nltk.corpus import brown

def pos_features(sentence,i,history):
    features = {"suffix(1)":sentence[i][-1:],
                "suffix(2)":sentence[i][-2:],
                "suffix(3)":sentence[i][-3:]}
    if i == 0:
        features["prev-word"]="<STAR>"
        features["prev_tag"] ="<STAR>"
    else:
        features["prev_word"]=sentence[i-1]
        features["prev_tag"]=history[i-1]        
    return features
    
class ConsecutivePosTagger(nltk.TaggerI):
    def __init__(self,train_sents):
        train_set=[]
        for tagged_sent in train_sents:
            history=[]
            untagged_sent = nltk.tag.untag(tagged_sent)
            for i,(word,tag) in enumerate(tagged_sent):
                featureset=pos_features(untagged_sent,i,history)
                train_set.append((featureset,tag))
                history.append(tag)
        self.classifier=nltk.NaiveBayesClassifier.train(train_set)
        
    def tag(self,sentence):
        history=[]
        for i,word in enumerate(sentence):
            featureset=pos_features(sentence,i,history)
            tag=self.classifier.classify(featureset)
            history.append(tag)
        return zip(sentence,history)
                
def test_ConsecutivePosTagger():
    tagged_sents=brown.tagged_sents(categories='news')
    size = int(len(tagged_sents) * 0.1)
    train_sents, test_sents = tagged_sents[size:], tagged_sents[:size]
    tagger = ConsecutivePosTagger(train_sents)
    
    print tagger.evaluate(test_sents)

流程为：

结果为：

0.796940194715