zoukankan      html  css  js  c++  java
  • go http爬虫

    1 
    
    package main
    
    import (
    	"fmt"
    	"io/ioutil"
    	"net/http"
    )
    
    func main() {
    	resp,err := http.Get("http://yeves.cn")
    	if err != nil{
    		fmt.Print("http get err",err)
    		return
    	}
    
    	body,err := ioutil.ReadAll(resp.Body)
    	if err != nil{
    		fmt.Print("http get err",err)
    		return
    	}
    
    	fmt.Print(string(body))
    }
    
    
    
    
    2
    package main
    
    import (
    	"fmt"
    	"io/ioutil"
    	"net/http"
    )
    
    func main() {
    	url := "http://yeves.cn"
    	download(url)
    }
    
    func download(url string) {
    	client := &http.Client{}
    	req,_ := http.NewRequest("GET",url,nil)
    
    	req.Header.Set("User-Agent","Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)")
    	resp,err := client.Do(req)
    	if err != nil{
    		fmt.Print("http get err",err)
    		return
    	}
    
    	defer resp.Body.Close()
    
    	body,err := ioutil.ReadAll(resp.Body)
    	if err != nil{
    		fmt.Print("read error ",err)
    		return
    	}
    	fmt.Print(string(body))
    }
    
    
    3
    
    package main
    
    import (
    	"fmt"
    	"github.com/jackdanger/collectlinks"
    	"net/http"
    )
    
    func main() {
    	url := "http://www.baidu.com/"
    	download(url)
    }
    
    func download(url string) {
    	client := &http.Client{}
    	req,_ := http.NewRequest("GET",url,nil)
    
    	req.Header.Set("User-Agent","Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)")
    	resp,err := client.Do(req)
    	if err != nil{
    		fmt.Print("http get err",err)
    		return
    	}
    
    	defer resp.Body.Close()
    
    	links := collectlinks.All(resp.Body)
    	for _,link := range links{
    		fmt.Print("parse url",link+"
    ")
    	}
    }
    

      

  • 相关阅读:
    c#格林治时间实现
    K3WISE常用表
    读取单元格数据
    水晶报表使用方法
    vs2010下使用sqlite
    C#执行EXE程序
    SQLLITE HELPER
    SQL LITE安装
    C#多线程
    VS2012 快捷键
  • 原文地址:https://www.cnblogs.com/brady-wang/p/13553729.html
Copyright © 2011-2022 走看看