[NLP自然语言处理]计算熵和KL距离，java实现汉字和英文单词的识别，UTF8变长字符读取

算法任务：

1. 给定一个文件，统计这个文件中所有字符的相对频率（相对频率就是这些字符出现的概率——该字符出现次数除以字符总个数，并计算该文件的熵）。

2. 给定另外一个文件，按上述同样的方法计算字符分布的概率，然后计算两个文件中的字符分布的KL距离。

（熵和KL距离都是NLP自然语言处理中术语，仅仅是涉及到一两个公式而已，不影响您对代码的理解，so just try！）

 

说明：

1. 给定的文件可以是两个中文文件或两个英文文件，也可以是两个中英文混合文件。对于中文，计算字符，对于英文，计算词。

2.有效字符不包括 空格 换行符 标点符号。

3.将中文字符、英文单词、其他非有效字符及其出现次数，分别写入三个文件中。

4.代码用java完成。

 

文章的重点：

1.如何判断一个字符是汉字，而不是ASCII、标点、日文、阿拉伯文……

2.了解汉字是如何编码的。“UTF8”绝逼是要花你一整个下午时间去弄明白的东西。

   推荐博文：http://www.cnblogs.com/chenwenbiao/archive/2011/08/11/2134503.html

3.正则表达式。对于计算机科班出身的人应该不陌生，在此我就不造次了。

 

代码如下：

import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map.Entry;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class NLPFileUnit {
    public HashMap<String, Integer> WordOccurrenceNumber;//The Occurrence Number of the single Chinese character
    //or Single English word in the file
    public HashMap<String, Float> WordProbability;//The probability of single Chinese character or English word
    public HashMap<String, Integer> Punctuations;//The punctuation that screened out from the file 
    public float entropy;//熵，本文主要计算单个汉字，或者单个英文单词的熵值
    private String filePath;

    //构造函数
    public NLPFileUnit(String filePath) throws Exception {
        this.filePath = filePath;
        WordOccurrenceNumber = createHash(createReader(filePath));
        Punctuations = filterPunctuation(WordOccurrenceNumber);
        WordProbability = calProbability(WordOccurrenceNumber);
        this.entropy = calEntropy(this.WordProbability);

        System.out.println("all punctuations were saved at " + filePath.replace(".", "_punctuation.") + "!");
        this.saveFile(Punctuations, filePath.replace(".", "_punctuation."));
        System.out.println("all words(En & Ch) were saved at " + filePath.replace(".", "_AllWords.") + "!");
        this.saveFile(this.WordOccurrenceNumber, filePath.replace(".", "_AllWords."));
    }

    /**
     * get the English words form the file to HashMap
     * @param hash
     * @param path
     * @throws Exception
     */
    public void getEnWords(HashMap<String, Integer> hash, String path) throws Exception {
        FileReader fr = new FileReader(path);
        BufferedReader br = new BufferedReader(fr);
        
        //read all lines into content
        String content = "";
        String line = null;
        while((line = br.readLine())!=null){
            content+=line;
        }
        br.close();
        
        //extract words by regex正则表达式
        Pattern enWordsPattern = Pattern.compile("([A-Za-z]+)");
        Matcher matcher = enWordsPattern.matcher(content);
        while (matcher.find()) {
            String word = matcher.group();
            if(hash.containsKey(word))
                hash.put(word, 1 + hash.get(word));
            else{
                hash.put(word, 1);
            }
        }
    }

    private boolean isPunctuation(String tmp) {
        //Punctuation should not be EN words/ Chinese
        final String cnregex = "\p{InCJK Unified Ideographs}";
        final String enregex = "[A-Za-z]+"; 
        return !(tmp.matches(cnregex) || tmp.matches(enregex)) ;
    }

    /**
     * judge whether the file is encoded by UTF-8 (UCS Transformation Format)format.
     * @param fs
     * @return
     * @throws Exception
     */
    private boolean isUTF8(FileInputStream fs) throws Exception {
        if (fs.read() == 0xEF && fs.read() == 0xBB && fs.read() == 0xBF)//所有utf8编码的文件前三个字节为0xEFBBBF
            return true;
        return false;
    }

    /**
     * utf8格式编码的字符，其第一个byte的二进制编码可以判断该字符的长度（汉字一般占三个字节）ASCII占一byte
     * @param b
     * @return
     */
    private int getlength(byte b) {
        int v = b & 0xff;//byte to 十六进制数
        if (v > 0xF0) {
            return 4;
        }
        // 110xxxxx
        else if (v > 0xE0) {
            return 3;
        } else if (v > 0xC0) {
            return 2;//该字符长度占2byte
        }
        return 1;
    }

    /**
     * 通过读取头一个byte来判断该字符占用字节数，并读取该字符，如1110xxxx，表示这个字符占三个byte
     * @param fs
     * @return
     * @throws Exception
     */
    private String readUnit(FileInputStream fs) throws Exception {
        byte b = (byte) fs.read();
        if (b == -1)
            return null;
        int len = getlength(b);
        byte[] units = new byte[len];
        units[0] = b;
        for (int i = 1; i < len; i++) {
            units[i] = (byte) fs.read();
        }
        String ret = new String(units, "UTF-8");
        return ret;
    }

    /**
     * 把单词，标点，汉字等全都读入hashmap
     * @param inputStream
     * @return
     * @throws Exception
     */
    private HashMap<String, Integer> createHash(FileInputStream inputStream)
            throws Exception {
        HashMap<String, Integer> hash = new HashMap<String, Integer>();
        String key = null;
        while ((key = readUnit(inputStream)) != null) {
            if (hash.containsKey(key)) {
                hash.put(key, 1 + (int) hash.get(key));
            } else {
                hash.put(key, 1);
            }
        }
        inputStream.close();
        getEnWords(hash, this.filePath);
        return hash;
    }

    /**
     * FileInputStream读取文件，若文件不是UTF8编码，返回null
     * @param path
     * @return
     * @throws Exception
     */
    private FileInputStream createReader(String path) throws Exception {
        FileInputStream br = new FileInputStream(path);
        if (!isUTF8(br))
            return null;
        return br;
    }

    /**
     * save punctuation filtered form (HashMap)hash into (HashMap)puncs,
     * @param hash;remove punctuation form (HashMap)hash at the same time
     * @return
     */
    private HashMap<String, Integer> filterPunctuation(
            HashMap<String, Integer> hash) {
        HashMap<String, Integer> puncs = new HashMap<String, Integer>();
        Iterator<?> iterator = hash.entrySet().iterator();

        while (iterator.hasNext()) {
            Entry<?, ?> entry = (Entry<?, ?>) iterator.next();
            String key = entry.getKey().toString();
            if (isPunctuation(key)) {
                puncs.put(key, hash.get(key));
                iterator.remove();
            }
        }
        return puncs;
    }

    /**
     * calculate the probability of the word in hash
     * @param hash
     * @return
     */
    private HashMap<String, Float> calProbability(HashMap<String, Integer> hash) {
        float count = countWords(hash);
        HashMap<String, Float> prob = new HashMap<String, Float>();
        Iterator<?> iterator = hash.entrySet().iterator();
        while (iterator.hasNext()) {
            Entry<?, ?> entry = (Entry<?, ?>) iterator.next();
            String key = entry.getKey().toString();
            prob.put(key, hash.get(key) / count);
        }
        return prob;
    }

    /**
     * save the content in the hash into file.txt
     * @param hash
     * @param path
     * @throws Exception
     */
    private void saveFile(HashMap<String, Integer> hash, String path)
            throws Exception {
        FileWriter fw = new FileWriter(path);
        fw.write(hash.toString());
        fw.close();
    }

    /**
     * calculate the total words in hash
     * @param hash
     * @return
     */
    private int countWords(HashMap<String, Integer> hash) {
        int count = 0;
        for (Entry<String, Integer> entry : hash.entrySet()) {
            count += entry.getValue();
        }
        return count;
    }

    /**
     * calculate the entropy（熵） of the characters
     * @param hash
     * @return
     */
    private float calEntropy(HashMap<String, Float> hash) {
        float entropy = 0;
        Iterator<Entry<String, Float>> iterator = hash.entrySet().iterator();
        while (iterator.hasNext()) {
            Entry<String, Float> entry = (Entry<String, Float>) iterator.next();
            Float prob = entry.getValue();//get the probability of the characters
            entropy += 0 - (prob * Math.log(prob));//calculate the entropy of the characters
        }
        return entropy;
    }
}







import java.io.BufferedReader;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map.Entry;

public class NLPWork {
 
    /**
     * calculate the KL distance form file u1 to file u2
     * @param u1
     * @param u2
     * @return
     */
    public static float calKL(NLPFileUnit u1, NLPFileUnit u2) {
        HashMap<String, Float> hash1 = u1.WordProbability;
        HashMap<String, Float> hash2 = u2.WordProbability;
        float KLdistance = 0;
        Iterator<Entry<String, Float>> iterator = hash1.entrySet().iterator();
        while (iterator.hasNext()) {
            Entry<String, Float> entry = iterator.next();
            String key = entry.getKey().toString();

            if (hash2.containsKey(key)) {
                Float value1 = entry.getValue();
                Float value2 = hash2.get(key);
                KLdistance += value1 * Math.log(value1 / value2);
            }
        }
        return KLdistance;
    }

    public static void main(String[] args) throws IOException, Exception {
        //all punctuation will be saved under working directory
        System.out.println("Now only UTF8 encoded file is supported!!!");
        System.out.println("PLS input file 1 path:");
        BufferedReader cin = new BufferedReader(
                new InputStreamReader(System.in));
        String file1 = cin.readLine();
        System.out.println("PLS input file 2 path:");
        String file2 = cin.readLine();
        NLPFileUnit u1 = null;
        NLPFileUnit u2 = null;
        try{
            u1 = new NLPFileUnit(file1);//NLP:Nature Language Processing
            u2 = new NLPFileUnit(file2);
        }
        catch(FileNotFoundException e){
            System.out.println("File Not Found!!");
            e.printStackTrace();
            return;
        }
        float KLdistance = calKL(u1, u2);
        System.out.println("KLdistance is :" + KLdistance);
        System.out.println("File 1 Entropy: " + u1.entropy);
        System.out.println("File 2 Entropy: " + u2.entropy);
    }
}

计算结果：