zoukankan      html  css  js  c++  java
  • stop_token.go

    package engine

    import (
        "bufio"
        "log"
        "os"
    )

    type StopTokens struct {
        stopTokens map[string]bool
    }

    // 从stopTokenFile中读入停用词,一个词一行
    // 文档索引建立时会跳过这些停用词
    func (st *StopTokens) Init(stopTokenFile string) {
        st.stopTokens = make(map[string]bool)
        if stopTokenFile == "" {
            return
        }

        file, err := os.Open(stopTokenFile)
        if err != nil {
            log.Fatal(err)
        }
        defer file.Close()

        scanner := bufio.NewScanner(file)
        for scanner.Scan() {
            text := scanner.Text()
            if text != "" {
                st.stopTokens[text] = true
            }
        }

    }

    func (st *StopTokens) IsStopToken(token string) bool {
        _, found := st.stopTokens[token]
        return found
    }

  • 相关阅读:
    CodeForces 834C
    HDU 6048
    HDU 6052
    HDU 6036
    HDU 6042
    HDU 2614 Beat(DFS)
    UESTC 1272 Final Pan's prime numbers(乱搞)
    HDU 2064 汉诺塔III(递归)
    HDU 2102 A计划(DFS)
    HDU 1069 I Think I Need a Houseboat(模拟)
  • 原文地址:https://www.cnblogs.com/zhangboyu/p/7461679.html
Copyright © 2011-2022 走看看