DotnetSpider爬虫简单示例 net core

文章地址

https://blog.csdn.net/sD7O95O/article/details/78097556

　

 安装爬虫框架  NUGET 安装DotnetSpider

创建HTTP协议数据包

            var site = new Site
            {
                CycleRetryTimes = 1,
                SleepTime = 200,
                Headers = new Dictionary<string, string>()
                {
                    {"Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8" },
                    {"Accept-Encoding","gzip, deflate, br" },
                    {"Accept-Language","zh-CN,zh;q=0.9" },
                    {"X-Requested-With","XMLHttpRequest" },
                    { "Referer", "https://blog.csdn.net/sD7O95O/article/details/78096027"},
                    { "Connection","keep-alive" },
                    { "Content-Type","text/html; charset=UTF-8" },
                    { "Host","blog.csdn.net"},
                    { "User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36"}
                }
            };
            site.AddStartUrl(surl);

创建一个爬虫

            Spider spider = Spider.Create(site,             // 
                new QueueDuplicateRemovedScheduler(),
                new CsdnArticleProcessor()                  //这个自定义的处理器
            ).AddPipeline(new CsdnArticlePipeline());       //自定义管道

　　

设置爬虫

　　　　　　  spider.Downloader = new HttpClientDownloader();
            spider.ThreadNum = 1;
            spider.EmptySleepTime = 3000;

　　

运行

 spider.Run();

　　

　

CsdnArticleProcessor 处理器  xpath 分析就不贴了

public class CsdnArticleProcessor : BasePageProcessor
    {
        protected override void Handle(Page page)
        {
            //page.Selectable.SelectList(Selectors.XPath("//table[@id='ip_list']/tr[2]/td[2]/text()")).Nodes();

            // 利用 Selectable 查询并构造自己想要的数据对象
            //xpath 得到自己需要的数据
　　　　　　　//...
　　　　　　　//page.AddResultItem("CountryResult", results);放入管道
        }
    }

    class CsdnArticlePipeline : BasePipeline
    {

        public override void Process(IEnumerable<ResultItems> resultItems, ISpider spider)
        {
　　　　　　var conlist = resukt.GetResultItem("CountryResult");获取数据
          //DB处理

        }
    }