.Net抓取网页数据

关键点

1、正则匹配有效数据

2、多线程并发获取数据

代码展示

class Crawler
    {
        /// <summary>
        /// URI
        /// </summary>
        public string Uri { get; set; }
        /// <summary>
        /// 创建实例时传入URI
        /// </summary>
        /// <param name="uri"></param>
        public Crawler(string uri)
        {
            this.Uri = uri;
        }
        /// <summary>
        /// 根据URI下载HTML
        /// </summary>
        /// <returns></returns>
        public string DownlodHtml()
        {
            //根据网站下载对应的html字符串
            using (WebClient wc = new WebClient())
            {
                wc.Encoding = Encoding.UTF8;
                string downloadStr = null;
                try
                {
                    downloadStr = wc.DownloadString(this.Uri);
                }
                catch(Exception e)
                {
                    downloadStr = null;
                    Console.WriteLine(e.Message);
                }
                return downloadStr;
            }
        }
        /// <summary>
        /// 获取数据
        /// </summary>
        /// <param name="regx"></param>
        /// <param name="html"></param>
        /// <param name="i">获取匹配的正则第几组数数据</param>
        /// <returns></returns>
        public List<string> GetData(string regx, string html, int i)
        {
            MatchCollection matches = Regex.Matches(html, regx);
            if (matches.Count > 0)
            {
                List<string> listTemp = new List<string>();
                foreach (Match item in matches)
                {
                    listTemp.Add(item.Groups[i].Value);
                }
                return listTemp;
            }
            else
            {
                return null;
            }
        }
    }

class MultiThreadCrawler
    {   
        private string _uri;
        //通知一个或多个正在等待的线程已发生事件
        private ManualResetEvent _doneEvent;

        public MultiThreadCrawler(string uri, ManualResetEvent doneEvent)
        {
            _uri = uri;
            _doneEvent = doneEvent;
        }

        public void ThreadPoolCallback(Object threadContext)
        {
            AutoGetData(_uri);
            //将事件设置为终止状态
            _doneEvent.Set();
        }

        public void ThreadPoolCallBackGetOnce(Object state)
        {
            AutoGetDataByPageNum(_uri);
            //将事件设置为终止状态
            _doneEvent.Set();
        }         
    }

参考资料
1、https://msdn.microsoft.com/zh-CN/library/3dasc8as.aspx