FP-Tree Java实现（二）：模板挖掘

从下往上，使用循环+递归模式识别日志模板。

package com.coshaho.fptree;

import java.util.*;
import java.util.stream.Collectors;

/**
 * FP树：仅考虑算法
 *
 * @author coshaho
 * @since 2020/1/5
 */
public class FPTree {
    // FP树根节点
    private FPNode root = new FPNode("Root", -1);
    // FP树节点线索头
    private Map<String, FPNode> firstNodeTable = new HashMap<>();
    // FP树节点线索尾
    private Map<String, FPNode> lastNodeTable = new HashMap<>();
    // 支持度
    private int support = 1;
    // 树的单词统计列表，降序
    private List<FPNode> table = new ArrayList<>();

    /**
     * 创建FP树
     * @param data 多行数据
     * @param count 每行数据出现次数
     * @param support 支持度
     */
    public FPTree(List<List<String>> data, List<Integer> count, int support) {
        this.support = support;
        if (null == count) {
            int size = data.size();
            count = new ArrayList<>();
            for (int i = 0; i < size; i++) {
                count.add(1);
            }
        }
        data = sort(data, count);
        // line为一行日志
        int i = 0;
        for (List<String> line : data) {
            FPNode curNode = root;
            for (String word : line) {
                if (curNode.getChildren().containsKey(word)) {
                    // 子节点存在则访问次数加一
                    curNode.getChildren().get(word).increase(count.get(i));
                } else {
                    // 子节点不存在则新增子节点
                    FPNode child = new FPNode(word, count.get(i));
                    curNode.getChildren().put(word, child);
                    child.setFather(curNode);
                }
                curNode = curNode.getChildren().get(word);
                // 当前节点有线索指向，则不必重复建立线索
                if (curNode.isVisited()) {
                    continue;
                }
                // 创建线索
                if (firstNodeTable.containsKey(word)) {
                    lastNodeTable.get(word).setNext(curNode);
                } else {
                    firstNodeTable.put(word, curNode);
                }
                lastNodeTable.put(word, curNode);
                curNode.setVisited(true);
            }
            i++;
        }
    }

    public void print() {
        root.print(0);
    }

    /**
     * 获取日志模板
     * @param last 下层节点
     */
    public void growth(List<String> last, List<LogTemplate> templates) {
        if (isSingleTree(this.root)) {
            getSingleTreeTemplate(last, templates);
        } else {
            getMultiTreeTemplate(last, templates);
        }
    }

    private void getWordTable(Map<String, Integer> wordCount) {
        for (Map.Entry<String, Integer> entry : wordCount.entrySet()) {
            if (entry.getValue() >= this.support) {
                table.add(new FPNode(entry.getKey(), entry.getValue()));
            }
        }
        if (0 != table.size()) {
            table = table.stream().sorted(Comparator.comparing(FPNode::getCount).reversed())
                    .collect(Collectors.toList());
        }
    }

    private Map<String, Integer> getWordCount(List<List<String>> data, List<Integer> count) {
        Map<String, Integer> wordCount = new HashMap<>();
        // 统计单词出现的次数
        int i = 0;
        for (List<String> line : data) {
            for (String word : line) {
                if (wordCount.containsKey(word)) {
                    wordCount.put(word, wordCount.get(word) + count.get(i));
                } else {
                    wordCount.put(word, count.get(i));
                }
            }
            i++;
        }
        return wordCount;
    }

    private List<List<String>> sortData(Map<String, Integer> wordCount, List<List<String>> data) {
        List<List<String>> result = new ArrayList<>();
        // 单词排序
        for (List<String> line : data) {
            List<String> newLine = line.stream()
                    .filter(word -> wordCount.get(word) >= support)
                    .sorted(Comparator.comparing(word -> wordCount.get(word)).reversed())
                    .collect(Collectors.toList());
            if (0 != newLine.size()) {
                result.add(newLine);
            }
        }
        return result;
    }

    private List<List<String>> sort(List<List<String>> data, List<Integer> count) {
        Map<String, Integer> wordCount = getWordCount(data, count);
        getWordTable(wordCount);
        return sortData(wordCount, data);
    }

    private void getSingleTreeTemplate(List<String> last, List<LogTemplate> templates) {
        // 获取单树路径上所有节点
        List<FPNode> wordCount = new ArrayList<>();
        FPNode child = getFirstChild(root);
        while (null != child) {
            wordCount.add(child);
            child = getFirstChild(child);
        }
        // 获取wordCount所有非空子集
        List<LogTemplate> sonTemplates = getSonSet(wordCount);
        for (LogTemplate template : sonTemplates) {
            // 子集合出现次数大于支撑度则保留为模板
            if (template.getCount() >= support) {
                templates.add(template);
                template.getWords().addAll(last);
            }
        }
    }

    private void getMultiTreeTemplate(List<String> last, List<LogTemplate> templates) {
        // table为树包含单词集合，降序
        // 此处转换为升序，从下往上计算以每个节点结尾的模板
        Collections.reverse(table);
        for (FPNode node : table) {
            List<String> curWords = new ArrayList<>();
            curWords.add(node.getWord());
            // last为上一层递归调用计算的节点
            curWords.addAll(last);
            // 当前节点当做一个日志模板
            if(null == last || 0 == last.size()) {
                LogTemplate template = new LogTemplate();
                template.setCount(node.getCount());
                List<String> words = new ArrayList<>();
                words.add(node.getWord());
                template.setWords(words);
                templates.add(template);
            }

            FPNode link = this.firstNodeTable.get(node.getWord());
            List<List<String>> data = new ArrayList<>();
            List<Integer> count = new ArrayList<>();
            // 一条线索上有多个节点，每个节点从下往上对应一条日志模板路径
            while (null != link) {
                FPNode me = link;
                List<String> meWords = new ArrayList<>();
                me = me.getFather();
                // 线索上每个节点往上走
                while (null != me.getFather()) {
                    meWords.add(me.getWord());
                    me = me.getFather();
                }
                count.add(link.getCount());
                // 不加这一句会导致排序不稳定
                Collections.reverse(meWords);
                data.add(meWords);
                link = link.getNext();
            }

            // 以上述节点构造新树
            FPTree newTree = new FPTree(data, count, this.support);
            newTree.growth(curWords, templates);
        }
    }

    private List<LogTemplate> getSonSet(List<FPNode> wordCount) {
        List<LogTemplate> result = new ArrayList<>();
        int length = wordCount.size();
        int mark;
        int nEnd = 1 << length;
        // 对于length位二进制数，每个数字对应一个子集合取法
        for (mark = 0; mark < nEnd; mark++) {
            LogTemplate template = new LogTemplate();
            // 循环查找每位是否应该放入集合
            for (int i = 0; i < length; i++) {
                //该位有元素输出
                if (((1 << i) & mark) != 0) {
                    template.getWords().add(wordCount.get(i).getWord());
                    // wordCount按照count降序排列，template count取最小值
                    template.setCount(wordCount.get(i).getCount());
                }
            }
            // 空集合舍弃
            if (template.getCount() != 0) {
                result.add(template);
            }
        }
        return result;
    }

    private boolean isSingleTree(FPNode tree) {
        if (null == tree || null == tree.getChildren() || 0 == tree.getChildren().size()) {
            return true;
        }
        // 有多个子节点则不是单树
        if (1 < tree.getChildren().size()) {
            return false;
        } else {
            return isSingleTree(getFirstChild(tree));
        }
    }

    private FPNode getFirstChild(FPNode tree) {
        if (null == tree || null == tree.getChildren() || 0 == tree.getChildren().size()) {
            return null;
        } else {
            for (FPNode child : tree.getChildren().values()) {
                return child;
            }
            return null;
        }
    }

    public static void main(String[] args) {
        List<String> line1 = new ArrayList<>();
        line1.add("C");
        line1.add("A");
        line1.add("B");
        List<String> line2 = new ArrayList<>();
        line2.add("A");
        line2.add("B");
        line2.add("D");
        List<String> line3 = new ArrayList<>();
        line3.add("A");
        line3.add("B");
        List<String> line4 = new ArrayList<>();
        line4.add("C");
        line4.add("E");
        List<List<String>> data = new ArrayList<>();
        data.add(line1);
        data.add(line2);
        data.add(line3);
        data.add(line4);

        FPTree tree = new FPTree(data, null, 1);
        tree.print();
        List<LogTemplate> templates = new ArrayList<>();
        tree.growth(new ArrayList<>(), templates);
        for (LogTemplate template : templates) {
            template.print();
        }
    }
}

package com.coshaho.fptree;

import java.util.HashMap;
import java.util.Map;

/**
 * FP树节点：仅考虑算法
 * @author coshaho
 * @since 2020/1/5
 */
public class FPNode {
    // 单词
    private String word;
    // 单词出现次数
    private int count = 1;
    // 子节点
    private Map<String, FPNode> children = new HashMap<>();
    // 父节点
    private FPNode father;
    // 线索：指向下一个相同单词节点
    private FPNode next;
    // 是否有线索指向自己
    private boolean visited = false;

    public FPNode(String word, int count) {
        this.word = word;
        this.count = count;
    }

    public void increase(int i) {
        count += i;
    }

    public void print(int n) {
        for(int i = 0; i < n; i++) {
            if(i == n - 1) {
                System.out.print("--");
            } else {
                System.out.print("  ");
            }
        }
        System.out.println(word + ": " + count);
        for(FPNode child : children.values()) {
            child.print(n + 1);
        }
    }

    public String getWord() {
        return word;
    }

    public void setWord(String word) {
        this.word = word;
    }

    public int getCount() {
        return count;
    }

    public void setCount(int count) {
        this.count = count;
    }

    public Map<String, FPNode> getChildren() {
        return children;
    }

    public void setChildren(Map<String, FPNode> children) {
        this.children = children;
    }

    public FPNode getFather() {
        return father;
    }

    public void setFather(FPNode father) {
        this.father = father;
    }

    public FPNode getNext() {
        return next;
    }

    public void setNext(FPNode next) {
        this.next = next;
    }

    public boolean isVisited() {
        return visited;
    }

    public void setVisited(boolean visited) {
        this.visited = visited;
    }
}

package com.coshaho.fptree;

import java.util.ArrayList;
import java.util.List;

/**
 * 日志模板
 *
 * @author coshaho
 * @since 2020/1/6
 */
public class LogTemplate {
    private List<String> words = new ArrayList<>();
    private int count;

    public List<String> getWords() {
        return words;
    }

    public void setWords(List<String> words) {
        this.words = words;
    }

    public int getCount() {
        return count;
    }

    public void setCount(int count) {
        this.count = count;
    }

    public void print() {
        System.out.println(words + ": " + count);
    }
}