c# 连续抓取页面内容

实现功能：去一个url抓取页面，在页面的内容里面在去找另一个url。找到这个这url之后经过一系列操作后再去重组的url去抓取内容。

第一、写出c#抓取页面的代码

using System;
using System.Collections.Generic;
using System.Linq;
using System.Web;
using System.IO;
using System.Net;
using System.Text;

/// <summary>
///abc 的摘要说明
/// </summary>
public static class abc
{
    /// <summary>
    /// webRequest 模拟http get请求
    /// </summary>
    /// <param name="strUrl">请求的url</param>
    /// <param name="encoding">编码</param>
    /// <returns>返回字符串</returns>
    public static string GetHttpResponse(this string strUrl, Encoding encoding)
    {
        string strResult = string.Empty;
        try
        {
            HttpWebRequest myReq = (HttpWebRequest)HttpWebRequest.Create(strUrl);
            HttpWebResponse HttpWResp = (HttpWebResponse)myReq.GetResponse();
            Stream myStream = HttpWResp.GetResponseStream();
            StreamReader sr = new StreamReader(myStream, encoding);
            strResult = sr.ReadToEnd();

        }
        catch (Exception ex)
        {
            WriteLog(ex.Message, strUrl);
        }

        return strResult;
    }

    /// <summary>
    /// webRequest 模拟http post请求
    /// </summary>
    /// <param name="url">请求的url</param>
    /// <param name="val">post 的数据</param>
    /// <returns>返回字符串</returns>
    public static string GetHttpPostResponse(this string url, string val, Encoding encoding)
    {
        string strResult = string.Empty;
        try
        {
            HttpWebRequest myReq = (HttpWebRequest)HttpWebRequest.Create(url);
            myReq.Method = "Post";
            myReq.ContentType = "application/x-www-form-urlencoded";
            byte[] byteArray = encoding.GetBytes(val);
            myReq.ContentLength = byteArray.Length;
            Stream stream = myReq.GetRequestStream();
            stream.Write(byteArray, 0, byteArray.Length);
            stream.Close();
            HttpWebResponse HttpWResp = (HttpWebResponse)myReq.GetResponse();
            Stream myStream = HttpWResp.GetResponseStream();
            StreamReader sr = new StreamReader(myStream, encoding);
            strResult = sr.ReadToEnd();

        }
        catch (Exception ex)
        {
            WriteLog(ex.Message + val, url);
        }

        return strResult;
    }

    public static void WriteLog(string sLog, string titleLog)
    {
        try
        {
            string logPath = System.AppDomain.CurrentDomain.BaseDirectory;//目录位置

            DateTime dt = DateTime.Now;
            string logfile = new StringBuilder(logPath).Append("\\Log\\").Append(dt.ToString("yyyy-MM-dd")).Append("\\").Append(titleLog).Append("_").Append(dt.ToString("yyyyMMddHHmmss")).Append(".txt").ToString();
            if (!System.IO.Directory.Exists(System.IO.Path.GetDirectoryName(logfile)))
            {
                System.IO.Directory.CreateDirectory(System.IO.Path.GetDirectoryName(logfile));
            }
            if (!File.Exists(logfile))
            {
                FileStream fs = System.IO.File.Create(logfile);
                fs.Close();
            }
            using (StreamWriter sw = new StreamWriter(logfile, true))
            {
                sw.WriteLine(DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss") + ":");
                sw.WriteLine(sLog);
                sw.WriteLine();
                sw.Close();
            }
        }
        catch
        {

        }
    }
}

第二、调用里面的方法GetHttpResponse去抓取页面（注这是get方式，如果是post方式可以选择post方式）

第三、用正则匹配方式得到想要的ＵＲＬ（Match mc = Regex.Match(aa, "action=(.*)>", RegexOptions.IgnoreCase);）

第四、由于此时得到URL是经过浏览器处理的URL如果我们直接去抓取页面就会找不到页面。因为此时的URL的协议是Https协议。所以我们需要中间做一次跳转。

首先还是去抓aa = abc.GetHttpResponse(str, Encoding.UTF8);得到的URL里有一个将要跳转到的url目录。我们需要将主域名+刚刚得到的这个目录。

第五、然后再去抓取。就可以得到我们想要的内容！