控制台爬取小说（大王饶命）

   var url = GetWBJokeUrl("/book/1719.html");
            string next;
            GetContent(url, out next);
            while (true)
            {
                ConsoleKeyInfo info = System.Console.ReadKey();
                switch (info.Key)
                {
                    case ConsoleKey.E:
                        Environment.Exit(0);
                        break;
                    case ConsoleKey.RightArrow:
                        GetContent(GetWBJokeUrl(next), out next);
                        break;
                    default:
                        System.Console.WriteLine(info.Key);
                        break;
                }

            }

 /// <summary>
        /// 获取大王饶命小说页面
        /// </summary>
        /// <param name="firstUrl">第一次进入的页面</param>
        /// <param name="nexturl">下一页</param>
        private static void GetContent(string firstUrl, out string nexturl)
        {
            var html = GetUrlContent(firstUrl);
            var url = @"<a href=([^>]+?)>下一页</a>";
            string re1 = "/.+html";
            nexturl = MatchReg(re1, MatchReg(url, html));
            var divContent = @"(?m)<div id=""BookText""[^>]*>(?<div>(?:w|W)*?)</div[^>]*>";
            html = MatchReg(divContent, html, "div").Trim().Replace("<br />", "");
            var delh4 = @"<h4>([sS]*?)</h4>";
            html = html.Replace(MatchReg(delh4, html), "");
            Console.WriteLine(html);

        }

        /// <summary>
        /// 筛选数据
        /// </summary>
        /// <param name="regStr">正则字符串</param>
        /// <param name="html">网页标签</param>
        /// <param name="input">需要获取的标签</param>
        /// <returns></returns>
        public static string MatchReg(string regStr, string html, string input = "0")
        {
            var reg = new Regex(regStr, RegexOptions.Multiline | RegexOptions.IgnoreCase);
            var mc = reg.Match(html);
            if (mc.Success)
            {
                return mc.Groups[input].Value;
            }
            return "";
        }

        /// <summary>
        /// 爬取地址
        /// </summary>
        const string qsbkMainUrl = "http://www.dawangraoming.com";
        /// <summary>
        /// 爬取页面位置
        /// </summary>
        /// <param name="firsturl"></param>
        /// <returns></returns>
        private static string GetWBJokeUrl(string firsturl)
        {
            StringBuilder url = new StringBuilder();
            url.Append(qsbkMainUrl);
            url.Append(firsturl);
            return url.ToString();
        }

        /// <summary>
        /// /伪装网站访问
        /// </summary>
        /// <param name="url">目标网站地址</param>
        /// <returns></returns>
        private static string GetUrlContent(string url)
        {
            try
            {

                HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);

                request.UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Maxthon/4.4.8.1000 Chrome/30.0.1599.101 Safari/537.36";

                request.Method = "GET";

                request.ContentType = "text/html;charset=UTF-8";

                HttpWebResponse response = (HttpWebResponse)request.GetResponse();

                Stream myResponseStream = response.GetResponseStream();

                StreamReader myStreamReader = new StreamReader(myResponseStream, Encoding.GetEncoding("utf-8"));

                string retString = myStreamReader.ReadToEnd();

                myStreamReader.Close();

                myResponseStream.Close();

                return retString;

            }

            catch { return null; }

        }