go爬虫

package main

import (
	"fmt"
	"github.com/antchfx/htmlquery"
	"io/ioutil"
	"net/http"
	"os"
	"regexp"
	"strings"
	"sync"
	"time"
)
var wg sync.WaitGroup
func main() {

	var url string = "https://haomooc.com/xiaoxue-read-2991.html"

	resp, _ := http.Get(url)

	defer resp.Body.Close()

	doc, _ := htmlquery.Parse(resp.Body)
	list := htmlquery.Find(doc, "//div[@class='dxs-l-b']//a")
	for _, li := range list {
		href := htmlquery.SelectAttr(li, "href")
		strings.Replace(href, " ", "", -1)
		title := htmlquery.SelectAttr(li,"title")
		strings.Replace(title, " ", "", -1)
		fmt.Printf("%s
", title)
		fmt.Printf("%s
", href)
		video := getVideo(href)
		wg.Add(1)
		//saveVideo(title,video)
		fmt.Printf("%s
", video)
	}

	wg.Wait()
}

func getVideo(url string) string  {
	time.Sleep(time.Second*1)
	resp, _ := http.Get(url)

	bytesContent, _ := ioutil.ReadAll(resp.Body)

	var reEmail = `(https://video.haomooc.com/.*.mp4)`

	re := regexp.MustCompile(reEmail)
	list := re.FindAllStringSubmatch(string(bytesContent), -1)

	var result string

	for _,v := range list {
		if v != nil{
			result = v[1]
		}
	}
	strings.Replace(result, " ", "", -1)
	return result
}
func PathExists(path string) (bool, error) {

	_, err := os.Stat(path)
	if err == nil {
		return true, nil
	}
	if os.IsNotExist(err) {
		return false, nil
	}
	return false, err
}

func saveVideo(title string ,url string)  {

	fmt.Printf(title,url)
	path := "/www/shell/video/"+title+".mp4"
	b, err := PathExists(path)
	if err != nil {
		fmt.Printf("PathExists(%s),err(%v)
", path, err)
	}
	if b {
		fmt.Printf("path %s 存在
", path)
	} else{
		fmt.Println("save video "+title )
		fmt.Printf("%s",url)
		// Get the data
		resp, err := http.Get(url)
		if err != nil {
			panic(err)
		}
		defer resp.Body.Close()

		data, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			panic(err)
		}
		ioutil.WriteFile(path, data, 0644)
		defer wg.Done()
	}


}