Lucene:如何写一个自定义的分词器

实现一个自定义分词器

实现一个简单的英文分词器，主要分为以下几个步骤：

1.建立自己的Attribute接口MyCharAttribute

/**
* MyCharAttribute
*
* @author limingcheng
* @Date 2019/11/28
*/
public interface MyCharAttribute extends Attribute {
void setChars(char[] buffer, int length);

char[] getChars();

int getLength();

String getString();
}

2.建立自定义attribute接口MyCharAttribute的实现类MyCharAttributeImpl

/**
* MyCharAttributeImpl
* 2.建立自定义attribute接口MyCharAttribute的实现类MyCharAttributeImpl
* 注意：MyCharAttributeImpl一定要和MyCharAttribute放在一个包下，否则会出现没有MyCharAttribute的实现类，
* 这是由org.apache.lucene.util.AttributeFactory.DefaultAttributeFactory.findImplClass(Class<? extends Attribute>)这个方法决定的
* @author limingcheng
* @Date 2019/11/28
*/
public class MyCharAttributeImpl extends AttributeImpl implements MyCharAttribute {

private char[] chatTerm = new char[255];
private int length = 0;

@Override
public void setChars(char[] buffer, int length) {
this.length = length;
if (length > 0) {
System.arraycopy(buffer, 0, this.chatTerm, 0, length);
}
}

@Override
public char[] getChars() {
return this.chatTerm;
}

@Override
public int getLength() {
return this.length;
}

@Override
public String getString() {
if (this.length > 0) {
return new String(this.chatTerm, 0, length);
}
return null;
}

@Override
public void clear() {
this.length = 0;
}

@Override
public void reflectWith(AttributeReflector reflector) {

}

@Override
public void copyTo(AttributeImpl target) {

}
}
3.建立分词器MyWhitespaceTokenizer：实现对英文按空白字符进行分词
/**
* MyWhitespaceTokenizer
*
* 3. 建立分词器MyWhitespaceTokenizer：实现对英文按空白字符进行分词
* @author limingcheng
* @Date 2019/11/28
*/
public class MyWhitespaceTokenizer extends Tokenizer {

// 需要记录的属性
// 词
MyCharAttribute charAttr = this.addAttribute(MyCharAttribute.class);

// 存词的出现位置

// 存放词的偏移

//
char[] buffer = new char[255];
int length = 0;
int c;

@Override
public boolean incrementToken() throws IOException {
// 清除所有的词项属性
clearAttributes();
length = 0;
while (true) {
c = this.input.read();

if (c == -1) {
if (length > 0) {
// 复制到charAttr
this.charAttr.setChars(buffer, length);
return true;
} else {
return false;
}
}

if (Character.isWhitespace(c)) {
if (length > 0) {
// 复制到charAttr
this.charAttr.setChars(buffer, length);
return true;
}
}

buffer[length++] = (char) c;
}
}

}

4.建立分项过滤器：把大写字母转换为小写字母

/**
* MyLowerCaseTokenFilter
*
* 4.建立分项过滤器：把大写字母转换为小写字母
* @author limingcheng
* @Date 2019/11/28
*/
public class MyLowerCaseTokenFilter extends TokenFilter {
public MyLowerCaseTokenFilter(TokenStream input) {
super(input);
}

MyCharAttribute charAttr = this.addAttribute(MyCharAttribute.class);

@Override
public boolean incrementToken() throws IOException {
boolean res = this.input.incrementToken();
if (res) {
char[] chars = charAttr.getChars();
int length = charAttr.getLength();
if (length > 0) {
for (int i = 0; i < length; i++) {
chars[i] = Character.toLowerCase(chars[i]);
}
}
}
return res;
}
}

5.建立分析器

/**
* MyWhitespaceAnalyzer
*
* 5. 建立分析器
* @author limingcheng
* @Date 2019/11/28
*/
public class MyWhitespaceAnalyzer extends Analyzer {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
Tokenizer source = new MyWhitespaceTokenizer();
TokenStream filter = new MyLowerCaseTokenFilter(source);
return new TokenStreamComponents(source, filter);
}

public static void main(String[] args) {

String text = "广州华为有限公司 An AttributeSource contains a list of different AttributeImpls, and methods to add and get them. ";

try {
Analyzer ana = new MyWhitespaceAnalyzer();
TokenStream ts = ana.tokenStream("aa", text);
MyCharAttribute ca = ts.getAttribute(MyCharAttribute.class);
ts.reset();
while (ts.incrementToken()) {
System.out.print(ca.getString() + "|");
}
ts.end();
ana.close();
System.out.println();
} catch (IOException e) {
e.printStackTrace();
}

}
}

一个简单的分词器可以这样实现，但是要实现一个可以对中文分词的分词器就需要算法方面的知识了。

本文参考：https://www.cnblogs.com/leeSmall/p/8993185.html