基于Lucene 4.x的ikanalyzer

需要修改IKAnalyzer.java、IKTokenizer.java、IKTokenizerFactory.java。

 1 import java.io.Reader;

 2 import org.apache.lucene.analysis.Analyzer;

 3 import org.apache.lucene.analysis.Tokenizer;

 4 

 5 /**

 6  * 实现Lucene Analyzer 基于IKTokenizer的中文分词器

 7  * 

 8  * @author 林良益

 9  * 

  */

 public final class IKAnalyzer extends Analyzer {

 

     private boolean isMaxWordLength = false;

 

     /**

      * IK分词器Lucene Analyzer接口实现类 默认最细粒度切分算法

      */

     public IKAnalyzer() {

         this(false);

     }

 

     /**

      * IK分词器Lucene Analyzer接口实现类

      * 

      * @param isMaxWordLength

      *            当为true时，分词器进行最大词长切分

      */

     public IKAnalyzer(boolean isMaxWordLength) {

         super();

         this.setMaxWordLength(isMaxWordLength);

     }

 

     @Override

     public TokenStreamComponents createComponents(String fieldName,

             Reader reader) {

         Tokenizer tokenizer = new IKTokenizer(reader, isMaxWordLength());

         return new TokenStreamComponents(tokenizer, null);

     }

 

     public void setMaxWordLength(boolean isMaxWordLength) {

         this.isMaxWordLength = isMaxWordLength;

     }

 

     public boolean isMaxWordLength() {

         return isMaxWordLength;

     }

 

 }

 1 import java.io.IOException;
 2 import java.io.Reader;
 3 
 4 import org.apache.lucene.analysis.Tokenizer;
 5 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
 6 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 7 import org.wltea.analyzer.IKSegmentation;
 8 import org.wltea.analyzer.Lexeme;
 9 
 /**
  * IK Analyzer v3.2 Lucene4.x Tokenizer适配器类 它封装了IKSegmentation实现
  * 
  * @author 林良益
  * 
  */
 public final class IKTokenizer extends Tokenizer {
     // IK分词器实现
     private IKSegmentation _IKImplement;
     // 词元文本属性
     private CharTermAttribute termAtt;
     // 词元位移属性
     private OffsetAttribute offsetAtt;
     // 记录最后一个词元的结束位置
     private int finalOffset;
 
     /**
      * Lucene Tokenizer适配器类构造函数
      * 
      * @param in
      * @param isMaxWordLength
      *            当为true时，分词器进行最大词长切分；当为false是，采用最细粒度切分
      */
     public IKTokenizer(Reader in, boolean isMaxWordLength) {
         super(in);
         offsetAtt = addAttribute(OffsetAttribute.class);
         termAtt = addAttribute(CharTermAttribute.class);
         _IKImplement = new IKSegmentation(in, isMaxWordLength);
     }
 
     @Override
     public final boolean incrementToken() throws IOException {
         // 清除所有的词元属性
         clearAttributes();
         Lexeme nextLexeme = _IKImplement.next();
         if (nextLexeme != null) {
             // 将Lexeme转成Attributes
             // 设置词元文本
             termAtt.setEmpty().append(nextLexeme.getLexemeText());
             // 设置词元位移
             offsetAtt.setOffset(nextLexeme.getBeginPosition(),
                     nextLexeme.getEndPosition());
             offsetAtt.setOffset(correctOffset(nextLexeme.getBeginPosition()), correctOffset(nextLexeme.getEndPosition()));
             finalOffset = nextLexeme.getEndPosition();
             // 返会true告知还有下个词元
             return true;
         }
         // 返会false告知词元输出完毕
         return false;
     }
 
     /*
      * (non-Javadoc)
      * 
      * @see org.apache.lucene.analysis.Tokenizer#reset(java.io.Reader)
      */
     public void reset() throws IOException {
         super.reset();
         _IKImplement.reset(input);
     }
 
     @Override
     public final void end() {
         offsetAtt.setOffset(finalOffset, finalOffset);
     }

 } 

 1 import java.io.Reader;
 2 import java.util.Map;
 3 
 4 import org.apache.lucene.analysis.Tokenizer;
 5 import org.apache.lucene.analysis.util.TokenizerFactory;
 6 import org.wltea.analyzer.lucene.IKTokenizer;
 7 
 8 /**
 9  * 实现Solr4.x分词器接口
10  * 基于IKTokenizer的实现
11  * 
12  * @author 林良益、李良杰
13  *
14  */
15 public final class IKTokenizerFactory extends TokenizerFactory{
16     
17     private boolean isMaxWordLength = false;
18     
19     /**
20      * IK分词器Solr TokenizerFactory接口实现类
21      * 默认最细粒度切分算法
22      */
23     public IKTokenizerFactory(){
24     }
25     
26     /*
27      * (non-Javadoc)
28      * @see org.apache.solr.analysis.BaseTokenizerFactory#init(java.util.Map)
29      */
30     public void init(Map<String,String> args){
31         String _arg = args.get("isMaxWordLength");
32         isMaxWordLength = Boolean.parseBoolean(_arg);
33     }
34     
35     /*
36      * (non-Javadoc)
37      * @see org.apache.solr.analysis.TokenizerFactory#create(java.io.Reader)
38      */
39     public Tokenizer create(Reader reader) {
40         return new IKTokenizer(reader , isMaxWordLength());
41     }
42 
43     public void setMaxWordLength(boolean isMaxWordLength) {
44         this.isMaxWordLength = isMaxWordLength;
45     }
46 
47     public boolean isMaxWordLength() {
48         return isMaxWordLength;
49     }
50 }