zoukankan      html  css  js  c++  java
  • 获取URL网页信息

            static string GetHtml(string url)
            {string strHTML = "";
                WebClient myWebClient = new WebClient();
                Stream myStream = myWebClient.OpenRead(url);
                StreamReader sr = new StreamReader(myStream, System.Text.Encoding.GetEncoding("utf-8"));
                strHTML = sr.ReadToEnd();
                myStream.Close();
                return strHTML;
            }
    
    
            static bool GetHtml(string url)
          {
        
                HttpWebRequest myHttpWebRequest = (HttpWebRequest)WebRequest.Create(url);
                myHttpWebRequest.Timeout = 20 * 1000; //连接超时
                myHttpWebRequest.Accept = "*/*";
                myHttpWebRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0;)";
                HttpWebResponse myHttpWebResponse = (HttpWebResponse)myHttpWebRequest.GetResponse();
                Stream stream = myHttpWebResponse.GetResponseStream();
                StreamReader sr =
                   new StreamReader(stream, Encoding.GetEncoding("utf-8"));
                var html = sr.ReadToEnd();
               
    }

    获取当前Cookies保存后获取整站信息

    static void Main(string[] args)
            {
               
    
                Console.WriteLine("确保挂载VPN后。。。回车键继续!");
                Console.ReadKey();
    
                string url = "http://www.sciencedirect.com/science/journal/00118486/61/6";
    
                string indata = "aa=zhuye";
                CookieContainer myCookieContainer =
                    new CookieContainer();
                //新建一个cookiecontainer来存放cookie集合
                HttpWebRequest myHttpWebRequest =
                    (HttpWebRequest)WebRequest.Create(url);
                //新建一个httpwebrequest
                myHttpWebRequest.ContentType = "application/x-www-form-urlencoded";
                myHttpWebRequest.ContentLength = indata.Length;
                myHttpWebRequest.Method = "post";
                myHttpWebRequest.CookieContainer = myCookieContainer;
                //设置httpwebrequest的cookiecontainer为
                //刚才建立的那个mycookiecontainer
                Stream myRequestStream = myHttpWebRequest.GetRequestStream();
                StreamWriter myStreamWriter = new StreamWriter(myRequestStream, Encoding.GetEncoding("gb2312"));
                myStreamWriter.Write(indata);
                //把数据写入httpwebrequest的request流
                myStreamWriter.Close();
                myRequestStream.Close();
                //关闭打开对象
                HttpWebResponse myHttpWebResponse =
                    (HttpWebResponse)myHttpWebRequest.GetResponse();
                //新建一个httpwebresponse
                myHttpWebResponse.Cookies = myCookieContainer.GetCookies(myHttpWebRequest.RequestUri);
              
                Console.WriteLine("获取cookies成功!。。。关闭VPN后任意键继续");
                Console.ReadKey();
    
                var htmlData = SaveCook(myHttpWebRequest, prevUrl, myCookieContainer, myHttpWebResponse);
                                   
                }
            }
    
     public static string SaveCook(HttpWebRequest myHttpWebRequest, string url, CookieContainer myCookieContainer, HttpWebResponse myHttpWebResponse)
            {
                //拿到了cookie,再进行请求就能直接读取到登录后的内容了
                myHttpWebRequest = (HttpWebRequest)WebRequest.Create(url);
                myHttpWebRequest.CookieContainer = myCookieContainer;//*
                                                                     //刚才那个cookiecontainer已经存有了cookie,把它附加到
     
                myHttpWebResponse = (HttpWebResponse)myHttpWebRequest.GetResponse();
                myHttpWebResponse.Cookies =
                    myCookieContainer.GetCookies(myHttpWebRequest.RequestUri);
    
    
                Stream myresponsestream = myHttpWebResponse.GetResponseStream();
                StreamReader mystreamreader =
                   new StreamReader(myresponsestream, Encoding.GetEncoding("gb2312"));
    
                var html = mystreamreader.ReadToEnd();
                //把数据从httpwebresponse的response流中读出
                mystreamreader.Close();
                myresponsestream.Close();
                return html;
    
            }
    
  • 相关阅读:
    KNN算法
    mysql必须知道的
    励志的演讲
    30条程序员名言警句
    别人的文章:为什么软件开发,人多,事少,还会工作量大?
    分享一个比较受启发的文章“学历代表过去,能力代表现在,学习力代表未来”
    mvc 学前必知
    启动docker容器 防火墙问题报错 ! -i docker0' failed: iptables: No chain/target/match by that name.
    git指令详解总结
    git reset 版本回退的三种用法总结
  • 原文地址:https://www.cnblogs.com/xuanlanbinfen/p/6889015.html
Copyright © 2011-2022 走看看