用lucene4.10.2分词器进行分词

import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.util.CharArraySet;

public class StandardAnalyzerTest {

	@SuppressWarnings("resource")
	public static void main(String[] args) {
		try {
			// 要处理的文本
            String text = "你好*hello,哈哈！ ！'ni> < !' hao->。“我”.192.168.8.10";
			// 自定义停用词		
			List<String> sw = new LinkedList<String>();// custom stopWords set
			sw.add("");
			CharArraySet stopWords = new CharArraySet(sw, true);			
			// 加入系统默认停用词
			Iterator<Object> itor = StandardAnalyzer.STOP_WORDS_SET.iterator();
			while (itor.hasNext()) {
				stopWords.add(itor.next());
			}
			// 标准分词器(Lucene内置的标准分析器,会将语汇单元转成小写形式，并去除停用词及标点符号)
			StandardAnalyzer analyzer = new StandardAnalyzer(stopWords);
			TokenStream ts = analyzer.tokenStream("field", text);
			CharTermAttribute ch = ts.addAttribute(CharTermAttribute.class);
			ts.reset();
			while (ts.incrementToken()) {
				System.out.println(ch.toString());
			}
			ts.end();
			ts.close();
		} catch (Exception ex) {
			ex.printStackTrace();
		}
	}

}
分词结果如下：
你
好
hello
哈
哈
ni
hao
我
192.168.8.10