zoukankan      html  css  js  c++  java
  • go http爬虫

    1 
    
    package main
    
    import (
    	"fmt"
    	"io/ioutil"
    	"net/http"
    )
    
    func main() {
    	resp,err := http.Get("http://yeves.cn")
    	if err != nil{
    		fmt.Print("http get err",err)
    		return
    	}
    
    	body,err := ioutil.ReadAll(resp.Body)
    	if err != nil{
    		fmt.Print("http get err",err)
    		return
    	}
    
    	fmt.Print(string(body))
    }
    
    
    
    
    2
    package main
    
    import (
    	"fmt"
    	"io/ioutil"
    	"net/http"
    )
    
    func main() {
    	url := "http://yeves.cn"
    	download(url)
    }
    
    func download(url string) {
    	client := &http.Client{}
    	req,_ := http.NewRequest("GET",url,nil)
    
    	req.Header.Set("User-Agent","Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)")
    	resp,err := client.Do(req)
    	if err != nil{
    		fmt.Print("http get err",err)
    		return
    	}
    
    	defer resp.Body.Close()
    
    	body,err := ioutil.ReadAll(resp.Body)
    	if err != nil{
    		fmt.Print("read error ",err)
    		return
    	}
    	fmt.Print(string(body))
    }
    
    
    3
    
    package main
    
    import (
    	"fmt"
    	"github.com/jackdanger/collectlinks"
    	"net/http"
    )
    
    func main() {
    	url := "http://www.baidu.com/"
    	download(url)
    }
    
    func download(url string) {
    	client := &http.Client{}
    	req,_ := http.NewRequest("GET",url,nil)
    
    	req.Header.Set("User-Agent","Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)")
    	resp,err := client.Do(req)
    	if err != nil{
    		fmt.Print("http get err",err)
    		return
    	}
    
    	defer resp.Body.Close()
    
    	links := collectlinks.All(resp.Body)
    	for _,link := range links{
    		fmt.Print("parse url",link+"
    ")
    	}
    }
    

      

  • 相关阅读:
    任务框架--Quartz 配置文件
    地址和值
    线性基学习笔记
    S07
    如何在实际项目中使用PageHelper分页插件
    设计模式:原型模式
    常用JS代码片段
    Thomson Plaza里面的三家店以及水果大会
    13.搜索过滤
    07-多线程
  • 原文地址:https://www.cnblogs.com/brady-wang/p/13553729.html
Copyright © 2011-2022 走看看