go 爬虫

 package main  

    import (  
        "fmt"  
        "io"  
        "io/ioutil"  
        "net/http"  
        "os"  
        "regexp"  
        "strconv"  
        "strings"  
        "time"  
    )  

    var (  
        // regHref       = `((ht|f)tps?)://[w]{0,3}.baidu.com/link?[a-zA-z=0-9-s]*`  
        regTitle      = `<title[sa-zA-z="-]*>([^x00-xff]|[sa-zA-Z=-：|，？"])*</title>`  
        regCheckTitle = `(为什么|怎么)*.*([G|g][O|o][L|l][A|a][N|n][G|g]).*(怎么|实现|如何|为什么).*`  
    )  

    func main() {  
       if checkFile("./data/", "url.txt").Size() == 0 {  
            fistStart()  
            main()  
        }         else {  
        Timer()  
        }  
    }  

    func Timer() {  
        t := time.NewTimer(time.Second * 1)  
        <-t.C  
        fmt.Print("


执行爬抓

")  
        f, _ := os.OpenFile("./data/url.txt", os.O_CREATE|os.O_APPEND|os.O_RDWR, 0666)  
        file, _ := ioutil.ReadAll(f)  
        pageCont, _ := pageVisit(strings.Split(string(file), "
")[0])  
        if checkRegexp(checkRegexp(pageCont, regTitle, 0).(string), regCheckTitle, 0).(string) != "" {  
            fmt.Print(checkRegexp(checkRegexp(pageCont, regTitle, 0).(string), regCheckTitle, 0).(string))  
            fmt.Print("
有效内容 => " + checkRegexp(pageCont, regTitle, 0).(string))  
        }  
        fmt.Print("

待爬抓网址共" + strconv.Itoa(len(strings.Split(string(file), "
"))-1) + "个 => " + strings.Split(string(file), "
")[0] + "
")  
        DelFirstText("./data/url.txt")  
        Timer()  
    }  

    func fistStart() {  
        var num int  
        url := "http://www.baidu.com/s?ie=utf-8&f=8&rsv_bp=1&tn=39042058_20_oem_dg&wd=golang%E5%AE%9E%E7%8E%B0&oq=golang%2520%25E5%2588%25A0%25E9%2599%25A4%25E6%2595%25B0%25E7%25BB%2584&rsv_pq=d9be28ec0002df1b&rsv_t=8017GWpSLPhDmKilZQ1StC04EVpUAeLEP90NIm%2Bk5pRh5R9o57NHMO8Gaxm1TtSOo%2FvtJj%2B98%2Fsc&rqlang=cn&rsv_enter=1&inputT=3474&rsv_sug3=16&rsv_sug1=11&rsv_sug7=100&rsv_sug2=0&rsv_sug4=4230"  
        resp, _ := http.Get(url)  
        defer resp.Body.Close()  
        body, _ := ioutil.ReadAll(resp.Body)  
        reg := regexp.MustCompile(`((ht|f)tps?)://[w]{0,3}.baidu.com/link?[a-zA-z=0-9-s]*`)  
        f, _ := os.OpenFile("./data/url.txt", os.O_CREATE|os.O_APPEND|os.O_RDWR, 0666)  
        defer f.Close()  
        for _, d := range reg.FindAllString(string(body), -1) {  
            ff, _ := os.OpenFile("./data/url.txt", os.O_RDWR, 0666)  
            file, _ := ioutil.ReadAll(ff)  
            dd := strings.Split(d, "")  
            dddd := ""  
            for _, ddd := range dd {  
                if ddd == "?" {  
                    ddd = `?`  
                }  
                dddd += ddd  
            }  
            if checkRegexp(string(file), dddd, 0).(string) == "" {  
                io.WriteString(f, d+"
")  
                fmt.Print("
收集地址：" + d + "
")  
                num++  
            }  
            // fmt.Print(string(file))  
            ff.Close()  
        }  
        fmt.Print("
首次收集网络地址：" + strconv.Itoa(len(reg.FindAllString(string(body), -1))) + "
")  
        fmt.Print("
去重后网络地址数：" + strconv.Itoa(num))  
        fmt.Print("

首次储存成功！
")  
    }  

    func pageVisit(url string) (page string, body []byte) {  
        resp, _ := http.Get(url)  
        defer resp.Body.Close()  
        body, _ = ioutil.ReadAll(resp.Body)  
        page = string(body)  
        return  
    }  

    func checkFile(dir string, file string) os.FileInfo {  
        list, _ := ioutil.ReadDir(dir)  
        for _, info := range list {  
            if info.Name() == file {  
                return info  
            }  
        }  
        return list[0]  
    }  

    func saveFile(file string, cont string) {  
        f, _ := os.OpenFile(file, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0666)  
        defer f.Close()  
        io.WriteString(f, cont)  
    }  

    func checkRegexp(cont string, reg string, style int) (result interface{}) {  
        check := regexp.MustCompile(reg)  
        switch style {  
        case 0:  
            result = check.FindString(cont)  
        case 1:  
            result = check.FindAllString(cont, -1)  
        default:  
            result = check.FindAll([]byte(cont), -1)  
        }  
        return  
    }  

    func DelFirstText(file string) {  
        var text = ""  
        f, _ := os.OpenFile(file, os.O_RDWR|os.O_CREATE, 0666)  
        files, _ := ioutil.ReadAll(f)  
        var ss = strings.Split(string(files), "
")  
        for i := 1; i < len(ss)-1; i++ {  
            text += ss[i] + "
"  
        }  
        defer f.Close()  
        ioutil.WriteFile(file, []byte(text), 0666)  
        fmt.Print("

删除该地址 => " + ss[0])  
    }