zoukankan      html  css  js  c++  java
  • 敏感词汇屏蔽

    @Service
    public class SensitiveService implements InitializingBean {
    
        private static final Logger logger = LoggerFactory.getLogger(SensitiveService.class);
    
        /**
         * 默认敏感词替换符
         */
        private static final String DEFAULT_REPLACEMENT = "敏感词";
    
    
        private class TrieNode {
    
            /**
             * true 关键词的终结 ; false 继续
             */
            private boolean end = false;
    
            /**
             * key下一个字符,value是对应的节点
             */
            private Map<Character, TrieNode> subNodes = new HashMap<>();
    
            /**
             * 向指定位置添加节点树
             */
            void addSubNode(Character key, TrieNode node) {
                subNodes.put(key, node);
            }
    
            /**
             * 获取下个节点
             */
            TrieNode getSubNode(Character key) {
                return subNodes.get(key);
            }
    
            boolean isKeywordEnd() {
                return end;
            }
    
            void setKeywordEnd(boolean end) {
                this.end = end;
            }
    
            public int getSubNodeCount() {
                return subNodes.size();
            }
    
    
        }
    
    
        /**
         * 根节点
         */
        private TrieNode rootNode = new TrieNode();
    
    
        /**
         * 判断是否是一个符号
         */
        private boolean isSymbol(char c) {
            int ic = (int) c;
            // 0x2E80-0x9FFF 东亚文字范围
            return !CharUtils.isAsciiAlphanumeric(c) && (ic < 0x2E80 || ic > 0x9FFF);
        }
    
    
        /**
         * 过滤敏感词
         */
        public String filter(String text) {
            if (StringUtils.isBlank(text)) {
                return text;
            }
            String replacement = DEFAULT_REPLACEMENT;
            StringBuilder result = new StringBuilder();
    
            TrieNode tempNode = rootNode;
            int begin = 0; // 回滚数
            int position = 0; // 当前比较的位置
    
            while (position < text.length()) {
                char c = text.charAt(position);
                // 空格直接跳过
                if (isSymbol(c)) {
                    if (tempNode == rootNode) {
                        result.append(c);
                        ++begin;
                    }
                    ++position;
                    continue;
                }
    
                tempNode = tempNode.getSubNode(c);
    
                // 当前位置的匹配结束
                if (tempNode == null) {
                    // 以begin开始的字符串不存在敏感词
                    result.append(text.charAt(begin));
                    // 跳到下一个字符开始测试
                    position = begin + 1;
                    begin = position;
                    // 回到树初始节点
                    tempNode = rootNode;
                } else if (tempNode.isKeywordEnd()) {
                    // 发现敏感词, 从begin到position的位置用replacement替换掉
                    result.append(replacement);
                    position = position + 1;
                    begin = position;
                    tempNode = rootNode;
                } else {
                    ++position;
                }
            }
    
            result.append(text.substring(begin));
    
            return result.toString();
        }
    
        private void addWord(String lineTxt) {
            TrieNode tempNode = rootNode;
            // 循环每个字节
            for (int i = 0; i < lineTxt.length(); ++i) {
                Character c = lineTxt.charAt(i);
                // 过滤空格
                if (isSymbol(c)) {
                    continue;
                }
                TrieNode node = tempNode.getSubNode(c);
    
                if (node == null) { // 没初始化
                    node = new TrieNode();
                    tempNode.addSubNode(c, node);
                }
    
                tempNode = node;
    
                if (i == lineTxt.length() - 1) {
                    // 关键词结束, 设置结束标志
                    tempNode.setKeywordEnd(true);
                }
            }
        }
    
    
        @Override
        public void afterPropertiesSet() throws Exception {
            rootNode = new TrieNode();
    
            try {
                InputStream is = Thread.currentThread().getContextClassLoader()
                        .getResourceAsStream("SensitiveWords.txt");
                InputStreamReader read = new InputStreamReader(is);
                BufferedReader bufferedReader = new BufferedReader(read);
                String lineTxt;
                while ((lineTxt = bufferedReader.readLine()) != null) {
                    lineTxt = lineTxt.trim();
                    addWord(lineTxt);
                }
                read.close();
            } catch (Exception e) {
                logger.error("读取敏感词文件失败" + e.getMessage());
            }
        }
    
        
        public static void main(String[] argv) {
            SensitiveService s = new SensitiveService();
            s.addWord("色情");
            s.addWord("好色");
            System.out.print(s.filter("你好X色**情XX"));
        }
    

    C#实现(生成dll调用即可,后续会更新去除空格的字符的屏蔽过滤测写法)

    ``` using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Collections; namespace com.lishuo.utils { public class SensitiveService { class TreeNode { //当前节点的所有子节点 //public HashTable subNodes = new HashTable(); public Hashtable subNodes = new Hashtable(); //判断当前树的结尾,即敏感词汇的结束 private bool end = false;//如果是true 就结束,否则继续往下判断
            //添加敏感词构造的的树
            public void addSubNodes(Char c, TreeNode node)
            {
                subNodes.Add(c, node);
            }
    
            //获取树节点
            public TreeNode getTreeNode(Char c)
            {
                return (TreeNode)subNodes[c];
            }
    
            //判断当前字符是否为敏感词
            public bool iskeyWords(Char c)
            {
                return end;
            }
    
            //设置敏感词的结束
            public void setKeyWords(bool end)
            {
                this.end = end;
            }
    
            //获取当前结点子节点的数量
            public int getSubNodeCount()
            {
                return subNodes.Count;
            }
    
        }
        TreeNode rootNode = new TreeNode();//建立树,空树
        /**
     * 将敏感词汇添加到树中
     */
        public void addWords(String TextLine)
        {
            TreeNode tempNode = rootNode;
    
            /* 这里面可以添加一些对传入字符串的过滤*/
            for (int i = 0; i < TextLine.Length; ++i)
            {
                Char c = TextLine[i];
                //TreeNode node = tempNode.getTreeNode(c);
                //if (node == null)
                //{//树的初始化
                TreeNode node = new TreeNode();
                tempNode.addSubNodes(c, node);
                //}
                tempNode = node;
                if (TextLine.Length - 1 == i)
                {
                    //如果敏感词汇结束,就是设置end为ture
                    tempNode.setKeyWords(true);
                }
            }
        }
        /**
        * 过滤敏感词的方法
        */
        public String filter(string message)
        {
    
            String REPLACEMENT = "*";//替换词
            TreeNode tempNode = rootNode;
            StringBuilder sb = new StringBuilder();//输出容器
    
            int begin = 0; //起始点
            int postion = 0;//当前活动节点
    
            //开始过滤
            while (postion < message.Length)
            {
                char c = message[postion];
                tempNode = tempNode.getTreeNode(c);
                if (tempNode == null)
                {
                    //说明当前匹配已经结束
                    sb.Append(c);
                    postion = begin + 1;
                    begin = postion;
                    tempNode = rootNode;
                }
                else if (tempNode.iskeyWords(c))
                {
                    for (int i = begin; i <= postion; ++i)
                    {
                        sb.Append(REPLACEMENT);
                    }
                    postion++;
                    begin = postion;
                }
                else
                {
                    ++postion;
                }
    
            }
            sb.Append(message.Substring(begin));
            return sb.ToString();
        }
    
    }
    

    }

  • 相关阅读:
    excel 读取
    MSDN异步编程概述 [C#] zzhttp://www.cnblogs.com/hxhbluestar/articles/60023.html
    window.opener showModelessDialog showModalDialog 获取|控制父窗体的区别
    MySql中文乱码解决方法
    关于随机数
    javascript 日期处理(注意事项)
    一个简单访问office程序的控件,不依赖officedll
    关于12306的bug
    回车提交
    js动态添加外部js(顶)
  • 原文地址:https://www.cnblogs.com/shuoli/p/7420380.html
Copyright © 2011-2022 走看看