zoukankan      html  css  js  c++  java
  • java读取中文分词工具(一)


    import java.io.BufferedReader;
    import java.io.File;
    import java.io.FileInputStream;
    import java.io.IOException;
    import java.io.InputStreamReader;
    import java.io.RandomAccessFile;
    import java.util.StringTokenizer;
    
    /*
     * 文本格式:已分词的中文文本,空格切割。有若干行。每行为一个段落。
     * 功能:遍历文档,逐个返回词语。
     * 两种模式:
     * 1 到文档末尾后,结束
     * 2 到文档末尾后。从头再读。

    / public class WordReader { static final int normalMode = 0; static final int againMode = 1; int currentMode = 0; //BufferedReader br=null; RandomAccessFile raf= null; StringTokenizer tokenizer = null; String nextWord=null; int currentLine = 0; int allCounts = 0; public WordReader(String fileName) throws IOException { File file=new File(fileName); //br=new BufferedReader(new InputStreamReader(new FileInputStream(file),"utf-8")); raf = new RandomAccessFile(file,"r") ; } private boolean hasNextWord() throws IOException { if( tokenizer!=null && tokenizer.hasMoreTokens()) { nextWord = tokenizer.nextToken(); return true; } else { String line=raf.readLine(); if(line == null) { if(currentMode == normalMode) return false; else //从头再来 { raf.seek(0); return hasNextWord();//递归 } } tokenizer = null; line = new String(line.getBytes("iso8859-1"),"utf-8"); tokenizer= new StringTokenizer(line," "); return hasNextWord();//递归 } } private String getNextWord() throws IOException { if(nextWord != null) { String word = nextWord; nextWord = null; allCounts ++; return word; } else if(hasNextWord()) { return getNextWord(); } else return null; } public static void main(String[] args) throws IOException { // TODO Auto-generated method stub WordReader wordReader = new WordReader("/home/linger/sources/ParaModel/electronic_seg.txt"); wordReader.currentMode = WordReader.againMode; //while(wordReader.hasNextWord())//共10329309个词 for(int i=0;i<10329319;i++)//文本从头读 { System.out.println(wordReader.getNextWord()); } System.out.println(wordReader.allCounts); } }



    用randomaccessfile类非常easy操作文件指针。

    可是遇到中文乱码问题。參考了这里http://blog.chinaunix.net/uid-15490606-id-211958.html。攻克了。

    line = new String(line.getBytes("iso8859-1"),"utf-8");

    对编码不是非常精通。有时见看看这个http://blog.sina.com.cn/s/blog_673c81990100t1lc.html。




  • 相关阅读:
    UOJ#80. 二分图最大权匹配 模板
    BZOJ2243: [SDOI2011]染色
    LA5713 Qin Shi Huang's National Road System
    BZOJ1977: [BeiJing2010组队]次小生成树 Tree
    LA5009 Error Curves
    BZOJ1013: [JSOI2008]球形空间产生器sphere
    BZOJ2733: [HNOI2012]永无乡
    BZOJ1552: [Cerc2007]robotic sort
    BZOJ3223: Tyvj 1729 文艺平衡树
    网络流24题(24/24)
  • 原文地址:https://www.cnblogs.com/cxchanpin/p/6912596.html
Copyright © 2011-2022 走看看