lucene 自定义分词器小程序

测试类
package LuceneUtil;

import java.io.Reader;
import java.util.Set;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.LetterTokenizer;
import org.apache.lucene.analysis.LowerCaseFilter;
import org.apache.lucene.analysis.StopAnalyzer;
import org.apache.lucene.analysis.StopFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.util.Version;
//自定义过滤分词器

public class MyStopAnalyzer extends Analyzer {
	private Set stops;
	public MyStopAnalyzer(String [] sws)//形参为 字符串数组
	{
		//会自动将字符串数组转换为Set
		stops=StopFilter.makeStopSet(Version.LUCENE_35, sws,true);
		//将原有的停用词加入到现在的停用词中
		stops.addAll(StopAnalyzer.ENGLISH_STOP_WORDS_SET);
	}
	
	public MyStopAnalyzer()
	{
		stops=StopAnalyzer.ENGLISH_STOP_WORDS_SET;
	}
	
	public TokenStream tokenStream(String FileName,Reader reader)
	{
		return new StopFilter(Version.LUCENE_35, 
				new LowerCaseFilter(Version.LUCENE_35,
				new LetterTokenizer(Version.LUCENE_35, reader)),  stops);
		
	}

}



 
package LuceneTest;

import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.StopAnalyzer;
import org.apache.lucene.util.Version;
import org.wltea.analyzer.lucene.IKAnalyzer;

import com.chenlb.mmseg4j.analysis.MMSegAnalyzer;

import LuceneUtil.AnalyzerUtils;
import LuceneUtil.MyStopAnalyzer;

public class TestAnalyzer {
	static ArrayList<String> list=null;
	
	
	public static void main (String [] args) throws IOException{
		//addNewWord( "烟台大学 ");
		//test();
		test01();
		
	}
	
	public static void test()
	{
		Analyzer a1=new MMSegAnalyzer();
		String txt="我是一名大学生，我来自菏*，我现在烟台大学。";
		AnalyzerUtils.displayToken(txt,a1); 
	
	}
	
	
	public static void test01()
	{
		//使用自定义的过滤分词器
		//这个语句 可以吧 “you“,”meet”,和“***”  给和谐掉
		Analyzer a2=new MyStopAnalyzer(new String [] {"you","meet","***"});
		//系统自带的StopAnalyzer
		Analyzer a3=new StopAnalyzer(Version.LUCENE_35);
		
		String txt=" i say :how are You,nice to meet you. ***";
		AnalyzerUtils.displayToken(txt,a2); 
		AnalyzerUtils.displayToken(txt,a3); 
	
	}
package LuceneUtil;
import java.io.IOException;
import java.io.StringReader;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.omg.CORBA.portable.Streamable;
//测试类

public class AnalyzerUtils {
	
	
	public static  void displayToken(String str,Analyzer a)
	{
		
		try {
			TokenStream ts=a.tokenStream("cotents", new StringReader(str));
			//创建一个属性，这个属性添加到流中，随着TokenStream增加
			CharTermAttribute cta=ts.addAttribute(CharTermAttribute.class);
			
			while (ts.incrementToken())
			{
				System.out.print("["+cta+"]");
				
			}
			System.out.println();
			
		} catch (IOException e) {
			
			e.printStackTrace();
		}
	}

}




/*public static void addNewWord(String newWord) throws IOException{BufferedWriter bw=new BufferedWriter(new FileWriter("G:\\mmseg\\data\\words-my.dic"));ArrayList<String> list=new ArrayList<String>();list.add(newWord);Iterator<String> iterator=list.iterator();while
 (iterator.hasNext()){bw.write(iterator.next());bw.flush();bw.newLine();}bw.close();System.out.println("添加成功");}*/}

测试结果如下：
可见 我想和谐掉的 那几个字已被和谐
第一行为执行和谐后的结果
第二行为未被和谐的

[i][say][how][nice]

[i][say][how][you][nice][meet][you][***]
不足：还不能对单个汉语词语和谐 ，汉语只能屏蔽一句话。而英语却可以