Lucene学习入门记录

前一段时间，公司让用Lucene检索文档，自己写了些代码，在这里记录一下，以免忘记了。

其实，简单的Lucene的入门还是很简单的，它的整体构造和关系型数据库差不多，一个键对应一个值，生成索引，然后根据索引去查找文档内容，在将内容通过别的方式显示出来。

Lucene创建、增、删、改索引：

package com.haiyisoft.szgl.file.service.impl;

import java.io.File;

/**
 * 档案管理的创建索引
 *
 * @author    haojiahong
 * 
 * <p>Modification History:</p>
 * <p>Date             Author      Description</p>
 * <p>--------------------------------------------------------------</p>
 * <p>20151027        haojiahong              new</p>
 * <p>  </p>
 */
@Component("schDocForDocBuilderService")
public class SchDocForDocBuilderServiceImpl implements SchDocForDocBuilderService {

	@Autowired
	public DocService docService;

	@Autowired
	public FileContentService fileContentService;

	private long time = 0;

	/**
	 * 创建lucene索引
	 */
	public void creatLucene() {
		IndexWriter indexWriter = null;
		try {
			File indexDir = new File(FileManage.searchCenterForDocPath);
			creatFile(indexDir);
			delAllFile(indexDir);
			Directory dir = FSDirectory.open(indexDir);
			Analyzer luceneAnalyzer = new IKAnalyzer();
			IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_36, luceneAnalyzer);
			indexWriter = new IndexWriter(dir, iwc);

			LogUtil.getAppLoger().debug("开始创建索引");

			long indexcount = this.createIndex(indexWriter);
			LogUtil.getAppLoger().debug("创建索引结束，共处理数据行数" + indexcount + "条");
			indexWriter.commit();
			indexWriter.close();
		} catch (IOException ex) {
			ex.printStackTrace();
		}
	}

	/**
	 * 按照数据集创建索引
	 * @param indexWriter
	 * @return
	 */
	private long createIndex(IndexWriter indexWriter) {
		try {
			this.showtime();
			long current = 0;
			// current += this.initFile(indexWriter);//根据文档建立索引
			current += this.initFileWithDocument(indexWriter);// 根据档案建立索引

			return current;
		} catch (Exception e) {
			e.printStackTrace();
			return -1;
		}
	}

	private long initFileWithDocument(IndexWriter indexWriter) {
		long current = 0;
		String jpql = "select file from FileManage file where 1=1";
		List<FileManage> fmLs = (List) JPAUtil.find(jpql);
		for (FileManage fm : fmLs) {
			try {
				indexWriter.addDocument(initLuceneDocument(indexWriter, current, fm));
				current++;
			} catch (Exception e) {
				e.printStackTrace();
			}
		}
		return current;
	}

	/**
	 * 增量添加 lucene索引
	 */
	@Override
	public void insertLucene(String fmUuid) {
		IndexWriter indexWriter = null;
		try {
			File indexDir = new File(FileManage.searchCenterForDocPath);
			Directory dir = FSDirectory.open(indexDir);
			Analyzer luceneAnalyzer = new IKAnalyzer();
			IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_36, luceneAnalyzer);
			indexWriter = new IndexWriter(dir, iwc);
			LogUtil.getAppLoger().debug("开始增量添加索引");
			long indexcount = this.insertIndex(indexWriter, fmUuid);
			LogUtil.getAppLoger().debug("添加索引结束，共处理数据行数" + indexcount + "条");
			indexWriter.commit();
			indexWriter.close();
		} catch (IOException ex) {
			ex.printStackTrace();
		}
	}

	/**
	 * 增量添加索引
	 * @param indexWriter
	 * @return
	 */
	private long insertIndex(IndexWriter indexWriter, String fmUuid) {
		try {
			this.showtime();
			long current = 0;
			current += this.insertFileWithDocument(indexWriter, fmUuid);// 根据档案添加索引
			return current;
		} catch (Exception e) {
			e.printStackTrace();
			return -1;
		}
	}

	private long insertFileWithDocument(IndexWriter indexWriter, String fmUuid) {
		long current = 0;
		FileManage fm = JPAUtil.loadById(FileManage.class, fmUuid);
		try {
			indexWriter.addDocument(initLuceneDocument(indexWriter, current, fm));
			current++;
		} catch (Exception e) {
			e.printStackTrace();
		}
		return current;
	}

	/**
	 * 更新Lucene索引
	 */
	@Override
	public void updateLucene(String fmUuid) {
		IndexWriter indexWriter = null;
		try {
			File indexDir = new File(FileManage.searchCenterForDocPath);
			Directory dir = FSDirectory.open(indexDir);
			Analyzer luceneAnalyzer = new IKAnalyzer();
			IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_36, luceneAnalyzer);
			indexWriter = new IndexWriter(dir, iwc);
			LogUtil.getAppLoger().debug("开始更新索引");
			long indexcount = this.updateIndex(indexWriter, fmUuid);
			LogUtil.getAppLoger().debug("更新索引结束，共处理数据行数" + indexcount + "条");
			indexWriter.commit();
			indexWriter.close();
		} catch (IOException ex) {
			ex.printStackTrace();
		}
	}

	/**
	 * 更新Lucene索引
	 */
	private long updateIndex(IndexWriter indexWriter, String fmUuid) {
		try {
			this.showtime();
			long current = 0;
			current += this.updateFileWithDocument(indexWriter, fmUuid);// 根据档案更新索引
			return current;
		} catch (Exception e) {
			e.printStackTrace();
			return -1;
		}
	}

	private long updateFileWithDocument(IndexWriter indexWriter, String fmUuid) {
		long current = 0;
		FileManage fm = JPAUtil.loadById(FileManage.class, fmUuid);
		try {
			indexWriter.updateDocument(new Term("UUID", fmUuid), initLuceneDocument(indexWriter, current, fm));
			current++;
		} catch (Exception e) {
			e.printStackTrace();
		}
		return current;
	}

	/**
	 * 删除Lucene索引
	 */
	@Override
	public void deteleLucene(String fmUuid) {
		IndexWriter indexWriter = null;
		try {
			File indexDir = new File(FileManage.searchCenterForDocPath);
			Directory dir = FSDirectory.open(indexDir);
			Analyzer luceneAnalyzer = new IKAnalyzer();
			IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_36, luceneAnalyzer);
			indexWriter = new IndexWriter(dir, iwc);
			LogUtil.getAppLoger().debug("开始删除索引");
			long indexcount = this.deteleIndex(indexWriter, fmUuid);
			LogUtil.getAppLoger().debug("删除索引结束，共处理数据行数" + indexcount + "条");
			indexWriter.commit();
			indexWriter.close();
		} catch (IOException ex) {
			ex.printStackTrace();
		}

	}

	/**
	 * 删除Lucene索引
	 * @param indexWriter
	 * @param fmUuid
	 * @return
	 */
	private long deteleIndex(IndexWriter indexWriter, String fmUuid) {
		try {
			this.showtime();
			long current = 0;
			current += this.deleteFileWithDocument(indexWriter, fmUuid);// 根据档案删除索引
			return current;
		} catch (Exception e) {
			e.printStackTrace();
			return -1;
		}
	}

	private long deleteFileWithDocument(IndexWriter indexWriter, String fmUuid) {
		long current = 0;
		try {
			indexWriter.deleteDocuments(new Term("UUID", fmUuid));
			current++;
		} catch (Exception e) {
			e.printStackTrace();
		}
		return current;
	}

	/**
	 * 批量删除索引
	 */
	@Override
	public void deteleLuceneLs(List<FileManage> fmList) {
		for (FileManage fm : fmList) {
			this.deteleLucene(fm.getUuid());
		}
	}

	/**
	 * 每个档案创建一个Lucene的document(创建、添加、更新索引用到此方法)
	 * @param indexWriter
	 * @param current
	 * @param fm
	 * @return
	 */
	private Document initLuceneDocument(IndexWriter indexWriter, long current, FileManage fm) {
		String fileName = "";
		String fileType = "";
		String fileContent = "";
		// TODO 版本号待定
		List<EdocFileObjectRelation> fileList = docService.queryFiles(DocConstant.NO_DIR, fm.getUuid(), fm.getClass()
				.getName(), null);
		if (fileList.size() > 0) {
			for (EdocFileObjectRelation file : fileList) {
				InputStream fs = docService.getFileInputStream(file.getFileId());
				fileName = file.getEdocFile().getName();
				fileType = fileName.substring(fileName.lastIndexOf(".") + 1);
				fileContent += FileManageUtil.getContent(fileType, fs);
				// String content = fileContentService(file.getEdocFile().)

			}
		}
		/**
		 * 写入索引文件
		 */
		Document doc = new Document();
		doc.add(new Field("UUID", fm.getUuid(), Store.YES, Index.NOT_ANALYZED));
		// doc.add(new Field("FILENAME", fileName, Store.YES,
		// Index.NO));// 文档名称
		if (!SzglCommonUtil.strIsNull(fileContent)) {
			doc.add(new Field("CONTENT", fileContent, Store.YES, Index.ANALYZED));// 具体内容
		}
		if (!SzglCommonUtil.strIsNull(fm.getTitle())) {
			doc.add(new Field("TITLE", fm.getTitle(), Store.YES, Index.ANALYZED));// 档案标题
		}
		if (!SzglCommonUtil.strIsNull(fm.getDocNum())) {
			doc.add(new Field("DOCNUM", fm.getDocNum(), Store.YES, Index.ANALYZED));// 档案文号
		}
		if (fm.getFromMan() != null) {
			doc.add(new Field("FROMMAN", fm.getFromMan().toString(), Store.YES, Index.ANALYZED));// 创建人
		}
		if (fm.getType() != null) {
			doc.add(new Field("TYPE", fm.getType(), Store.YES, Index.ANALYZED));// 档案类别
		}
		if (fm.getStatus() != null) {
			doc.add(new Field("STATUS", fm.getStatus(), Store.YES, Index.ANALYZED));// 档案状态
		}

		if (!SzglCommonUtil.strIsNull(fm.getIsShare())) {
			doc.add(new Field("ISHARE", fm.getIsShare(), Store.YES, Index.ANALYZED));// 档案是否共享
		}
		if (fm.getFromTime() != null) {
			doc.add(new Field("FROMTIME", fm.getFromTime() + "", Store.YES, Index.ANALYZED));// 档案传来时间
		}
		if (!SzglCommonUtil.strIsNull(fm.getBoxUuid())) {
			FileBox filebox = JPAUtil.loadById(FileBox.class, fm.getBoxUuid());
			if (!SzglCommonUtil.strIsNull(filebox.getYearCode())) {
				doc.add(new Field("YEARCODE", filebox.getYearCode(), Store.YES, Index.ANALYZED));// 档案所属的档案盒年度
			}
		}

		return doc;
	}

	private boolean delAllFile(File file) {
		boolean flag = false;
		if (file != null) {
			File[] tempList = file.listFiles();
			File temp = null;
			for (int i = 0; i < tempList.length; i++) {
				temp = tempList[i];
				if (temp.isFile()) {
					temp.delete();
				}
			}
		}
		return flag;
	}

	/**
	 * 显示时间
	 */
	private void showtime() {
		long time1 = System.currentTimeMillis();
		if (time > 0) {
			LogUtil.getAppLoger().debug("MilliSecond:" + (time1 - time));
		} else {
			LogUtil.getAppLoger().debug("Start time:" + (new Timestamp(System.currentTimeMillis())));
		}
		time = time1;
	}

	private void creatFile(File file) {
		if (!file.exists()) {
			file.mkdirs();
		}
	}

	private long initFile(IndexWriter indexWriter) {
		long current = 0;
		String jpql = "select file from FileManage file where 1=1";
		List<FileManage> fmLs = (List) JPAUtil.find(jpql);
		for (FileManage fm : fmLs) {
			// TODO 版本号待定
			List<EdocFileObjectRelation> fileList = docService.queryFiles(DocConstant.NO_DIR, fm.getUuid(), fm
					.getClass().getName(), null);
			if (fileList.size() > 0) {
				for (EdocFileObjectRelation file : fileList) {
					InputStream fs = docService.getFileInputStream(file.getFileId());
					String fileName = file.getEdocFile().getName();
					String fileType = fileName.substring(fileName.lastIndexOf(".") + 1);
					String fileContent = FileManageUtil.getContent(fileType, fs);
					// String content = fileContentService(file.getEdocFile().)
					/**
					 * 写入索引文件
					 */
					try {
						Document doc = new Document();
						// doc.add(new Field("TYPE", fm.getType(), Store.YES,
						// Index.ANALYZED)); // 档案类别
						doc.add(new Field("UUID", fm.getUuid(), Store.YES, Index.NO));
						doc.add(new Field("FILENAME", fileName, Store.YES, Index.NO));// 文档名称
						if (!SzglCommonUtil.strIsNull(fileContent)) {
							doc.add(new Field("CONTENT", fileContent, Store.YES, Index.ANALYZED));// 具体内容
						}
						if (!SzglCommonUtil.strIsNull(fm.getTitle())) {
							doc.add(new Field("TITLE", fm.getTitle(), Store.YES, Index.ANALYZED));// 档案标题
						}
						if (!SzglCommonUtil.strIsNull(fm.getDocNum())) {
							doc.add(new Field("DOCNUM", fm.getDocNum(), Store.YES, Index.ANALYZED));// 档案文号
						}
						if (fm.getFromMan() != null) {
							doc.add(new Field("FROMMAN", fm.getFromMan().toString(), Store.YES, Index.ANALYZED));// 创建人
						}

						indexWriter.addDocument(doc);
						current++;
						if ((current - (current / 10000) * 10000) == 0) {
							LogUtil.getAppLoger().debug("current row num:" + current);
						}
					} catch (Exception ex) {
						ex.printStackTrace();
					}
				}
			}

		}
		return current;
	}
}

根据索引去查询，并将关键词标红

package com.haiyisoft.szgl.file.service.impl;

import java.io.File;

/**
 * 通过Lucene查询
 *
 * @author    haojiahong
 * 
 * <p>Modification History:</p>
 * <p>Date             Author      Description</p>
 * <p>--------------------------------------------------------------</p>
 * <p>20151102        haojiahong              new</p>
 * <p>  </p>
 */

@Component("fileSchByLuceneService")
public class FileSchByLuceneServiceImpl implements FileSchByLuceneService {

	@Override
	public List<FileManage> retrieveByLucene(String keyword, String titleSch, String docNumSch, String typeSch,
			String isShareSch, Timestamp yearCodeBegin, Timestamp yearCodeEnd, SortParamList sortParamList,
			PageInfo pageInfo) {
		List<FileManage> result = new ArrayList<FileManage>();
		IndexSearcher searcher = null;
		String indexDir = FileManage.searchCenterForDocPath;
		File file = new File(indexDir);
		if ((SzglCommonUtil.strIsNull(keyword) && SzglCommonUtil.strIsNull(docNumSch)
				&& SzglCommonUtil.strIsNull(titleSch) && SzglCommonUtil.strIsNull(typeSch) && SzglCommonUtil
					.strIsNull(isShareSch)) || (!file.exists())) {
			if (pageInfo != null) {
				pageInfo.setAllRowNum(0);
			}
			return null;
		}

		try {
			Directory dir = FSDirectory.open(new File(indexDir));
			IndexReader reader = IndexReader.open(dir);
			searcher = new IndexSearcher(reader);
			BooleanQuery query = new BooleanQuery();

			Analyzer anal = new IKAnalyzer();
			QueryParser qp = new QueryParser(Version.LUCENE_36, "CONTENT", anal);
			QueryParser qpTitle = new QueryParser(Version.LUCENE_36, "TITLE", anal);
			QueryParser qpDocNum = new QueryParser(Version.LUCENE_36, "DOCNUM", anal);
			QueryParser qpIshare = new QueryParser(Version.LUCENE_36, "ISHARE", anal);
			if (!SzglCommonUtil.strIsNull(keyword)) {
				query.add(qp.parse(keyword), Occur.MUST);
			}
			if (!SzglCommonUtil.strIsNull(titleSch)) {
				query.add(qpTitle.parse(titleSch), Occur.MUST);
			}
			if (!SzglCommonUtil.strIsNull(docNumSch)) {
				query.add(qpDocNum.parse(docNumSch), Occur.MUST);
			}
			if (!SzglCommonUtil.strIsNull(typeSch)) {
				query.add(new TermQuery(new Term("TYPE", typeSch)), Occur.MUST);
			}
			if (!SzglCommonUtil.strIsNull(isShareSch)) {
				query.add(qpIshare.parse(isShareSch), Occur.MUST);
			}
			if (yearCodeBegin != null || yearCodeEnd != null) {
				query.add(new TermRangeQuery("YEARCODE", yearCodeBegin.toString(), yearCodeEnd.toString(), true, true),
						Occur.MUST);
			}
			ScoreDoc[] hits = searcher.search(query, Integer.MAX_VALUE).scoreDocs;
			int nowPagestart = (pageInfo.getCurPageNum() - 1) * pageInfo.getRowOfPage();// 当前页第一条数据是总数的第几条数据
			int allPage = hits.length;// 总条数
			pageInfo.setAllRowNum(allPage);
			int nowPageEnd = (nowPagestart + pageInfo.getRowOfPage()) < allPage ? (nowPagestart + pageInfo
					.getRowOfPage()) : allPage;
			for (int i = nowPagestart; i < nowPageEnd; i++) {
				FileManage fm = new FileManage();
				Document doc = searcher.doc(hits[i].doc);
				fm.setUuid(doc.get("UUID"));
				if (!SzglCommonUtil.strIsNull(docNumSch)) {
					fm.setDocNum(lighterStr(anal, qpDocNum.parse(docNumSch), doc.get("DOCNUM"), "DOCNUM"));
				} else {
					fm.setDocNum(doc.get("DOCNUM"));
				}
				if (!SzglCommonUtil.strIsNull(titleSch)) {
					fm.setTitle(lighterStr(anal, qpTitle.parse(titleSch), doc.get("TITLE"), "TITLE"));
				} else {
					fm.setTitle(doc.get("TITLE"));
				}
				if (!SzglCommonUtil.strIsNull(keyword)) {
					fm.setFileContent(lighterStr(anal, qp.parse(keyword), doc.get("CONTENT"), "CONTENT"));
				} else {
					fm.setFileContent(doc.get("CONTENT"));
				}
				fm.setFromMan(Long.valueOf(doc.get("FROMMAN")));
				fm.setFileName(doc.get("FILENAME"));
				fm.setType(doc.get("TYPE"));
				fm.setStatus(doc.get("STATUS"));
				fm.setFromTime(Timestamp.valueOf(doc.get("FROMTIME")));
				result.add(fm);
			}
		} catch (IOException e) {
			e.printStackTrace();
		} catch (Exception e) {
			e.printStackTrace();
		} finally {
			if (searcher != null)
				try {
					searcher.close();
				} catch (IOException e) {
					e.printStackTrace();
				}
		}

		return result;
	}

	private String lighterStr(Analyzer a, Query query, String txt, String fieldname) throws Exception {
		String str = null;
		QueryScorer scorer = new QueryScorer(query);
		Fragmenter fragmenter = new SimpleSpanFragmenter(scorer);
		Formatter fmt = new SimpleHTMLFormatter("<span style='color:red'>", "</span>");
		Highlighter lighter = new Highlighter(fmt, scorer);
		lighter.setTextFragmenter(fragmenter);
		str = lighter.getBestFragment(a, fieldname, txt);
		if (str == null)
			return txt;
		return str;
	}

}

　　一些工具类，用于读文档内容。

 * Copyright (C) 2014-2020 Yantai HaiYi Software Co.,Ltd
package com.haiyisoft.szgl.file.util;

import java.io.BufferedReader;

/**
 * 档案管理工具类
 *
 * @author    haojiahong
 * 
 * <p>Modification History:</p>
 * <p>Date             Author      Description</p>
 * <p>--------------------------------------------------------------</p>
 * <p>20151102       haojiahong              new</p>
 * <p>  </p>
 */
public class FileManageUtil {

	/**
	 * 根据文件类型，获取文本内容
	 * @param type
	 * @param fs
	 * @return
	 */
	public static String getContent(String type, InputStream fs) {
		String text = null;
		if ("doc".equals(type)) {
			POITextExtractor ex = null;
			try {
				ex = new WordExtractor(fs);
				text = ex.getText();
			} catch (Exception e) {
				e.printStackTrace();
			}
		} else if ("docx".equals(type)) {
			POITextExtractor ex = null;
			try {
				OPCPackage opcPackage = OPCPackage.open(fs);
				ex = new XWPFWordExtractor(opcPackage);
				text = ex.getText();
			} catch (Exception e) {
				e.printStackTrace();
			}
		} else if ("txt".equals(type)) {
			BufferedReader reader = new BufferedReader(new InputStreamReader(fs));
			String line = null;
			try {
				while ((line = reader.readLine()) != null) {
					text += line;
				}
			} catch (IOException e) {
				e.printStackTrace();
			}
		} else if ("pdf".equals(type)) {
			try {
				PDDocument pdfDocument = PDDocument.load(fs);
				text = new PDFTextStripper().getText(pdfDocument);
			} catch (IOException e) {
				e.printStackTrace();
			}
		}

		return text;
	}
}