【原创】Scrapyd 的 .net 客户端

最近项目需要部署Scrapy爬虫，采用最简单的Scrapyd服务进行部署，基于.net core 进行了客户端的封装。

1）Scrapyd API文档：http://scrapyd.readthedocs.io/en/latest/api.html 

2）使用HttpClient进行交互

比较麻烦的是部署爬虫所需要的使用 HttpClient 上传文件的代码，使用了  “MultipartFormDataContent”  对象进行上传，添加字符串内容，及StreamContent，直接可以将MVC中的File对象传入，不用自行处理临时文件。

 var content = new MultipartFormDataContent("Upload-----" + Guid.NewGuid().ToString().Replace("-", ""));

                content.Add(new StringContent(project), ""project"");
                content.Add(new StringContent(version), ""version"");
                content.Add(new StreamContent(eggStream), ""egg"", """ + project + ".egg" + """);

                var resp = client.PostAsync("addversion.json", content).Result;

                var strResult = resp.Content.ReadAsStringAsync().Result;

                return JsonConvert.DeserializeObject<AddVersionResponse>(strResult);

完整代码

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Text;
using Newtonsoft.Json;
using Scrapyd.Client.Response;

namespace Scrapyd.Client
{
    /// <summary>
    /// A Client used to communicate with the Scrapyd Daemon
    /// </summary>
    public class ScrapydClient
    {
        private readonly string _host;
        private readonly int _port;

        public ScrapydClient(string host, int port)
        {
            _host = host;
            _port = port;
        }

        /// <summary>
        /// 查询状态
        /// </summary>
        /// <returns></returns>
        public DaemonStatusResponse QueryStatus()
        {
            HttpClient client = new HttpClient()
            {
                BaseAddress = new Uri($"http://{_host}:{_port}")
            };

            try
            {
                var strResult = client.GetStringAsync("daemonstatus.json").Result;

                var jresult = JsonConvert.DeserializeObject<DaemonStatusResponse>(strResult);
                return jresult;
            }
            catch (Exception e)
            {
                return new DaemonStatusResponse { status = "failed", message = e.Message };
            }
        }

        public AddVersionResponse AddVersion(string project, string version, Stream eggStream)
        {
            if (string.IsNullOrEmpty(project)) throw new ArgumentNullException(nameof(project));
            if (eggStream == null) throw new ArgumentNullException(nameof(eggStream));

            var client = new HttpClient()
            {
                BaseAddress = new Uri($"http://{_host}:{_port}")
            };

            try
            {
                var content = new MultipartFormDataContent("Upload-----" + Guid.NewGuid().ToString().Replace("-", ""));

                content.Add(new StringContent(project), ""project"");
                content.Add(new StringContent(version), ""version"");
                content.Add(new StreamContent(eggStream), ""egg"", """ + project + ".egg" + """);

                var resp = client.PostAsync("addversion.json", content).Result;

                var strResult = resp.Content.ReadAsStringAsync().Result;

                return JsonConvert.DeserializeObject<AddVersionResponse>(strResult);
            }
            catch (Exception e)
            {
                return new AddVersionResponse { status = "failed", message = e.Message };
            }
        }

        public ScheduleResponse ScheduleJob(string project, string spider,
            string setting = "",
            string jobid = "",
            string version = "",
            Dictionary<string, string> arguments = null)
        {
            if (string.IsNullOrEmpty(project)) throw new ArgumentNullException(nameof(project));
            if (string.IsNullOrEmpty(spider)) throw new ArgumentNullException(nameof(spider));


            var client = new HttpClient()
            {
                BaseAddress = new Uri($"http://{_host}:{_port}")
            };

            try
            {
                var data = new Dictionary<string, string>()
                {
                    {"project", project},
                    {"spider", spider}
                };

                if (!string.IsNullOrEmpty(jobid))
                {
                    data.Add("jobid", jobid);
                }
                if (!string.IsNullOrEmpty(setting))
                {
                    data.Add("setting", setting);
                }

                if (!string.IsNullOrEmpty(version))
                {
                    data.Add("_version", version);
                }

                if (arguments != null && arguments.Count > 0)
                {
                    foreach (var argument in arguments)
                    {
                        data.Add(argument.Key, argument.Value);
                    }
                }

                var strResult =
                    client.PostAsync("schedule.json", new FormUrlEncodedContent(data))
                        .Result.Content.ReadAsStringAsync()
                        .Result;

                return JsonConvert.DeserializeObject<ScheduleResponse>(strResult);

            }
            catch (Exception e)
            {
                return new ScheduleResponse { status = "failed", message = e.Message };
            }
        }

        public CancelResponse CancelJob(string project, string jobid)
        {
            if (string.IsNullOrEmpty(project)) throw new ArgumentNullException(nameof(project));
            if (string.IsNullOrEmpty(jobid)) throw new ArgumentNullException(nameof(jobid));


            var client = new HttpClient()
            {
                BaseAddress = new Uri($"http://{_host}:{_port}")
            };

            try
            {
                var content = new FormUrlEncodedContent(new Dictionary<string, string>()
                {
                    {"project", project},
                    {"job", jobid}
                });
                var strResult = client.PostAsync($"/cancel.json", content).Result.Content.ReadAsStringAsync().Result;

                return JsonConvert.DeserializeObject<CancelResponse>(strResult);

            }
            catch (Exception e)
            {
                return new CancelResponse { status = "failed", message = e.Message };
            }
        }

        public ListProjectsResponse ListProjects()
        {
            try
            {
                var client = new HttpClient
                {
                    BaseAddress = new Uri($"http://{_host}:{_port}")
                };
                var result = client.GetStringAsync("listprojects.json").Result;
                return JsonConvert.DeserializeObject<ListProjectsResponse>(result);
            }
            catch (Exception e)
            {
                return new ListProjectsResponse { status = "failed", message = e.Message };
            }
        }

        public ListVersionsResponse ListVersions(string project)
        {
            if (string.IsNullOrEmpty(project)) throw new ArgumentNullException(nameof(project));

            try
            {
                var client = new HttpClient
                {
                    BaseAddress = new Uri($"http://{_host}:{_port}")
                };

                var result = client.GetStringAsync($"listversions.json?project={project}").Result;
                return JsonConvert.DeserializeObject<ListVersionsResponse>(result);
            }
            catch (Exception e)
            {
                return new ListVersionsResponse { status = "failed", message = e.Message };
            }
        }

        public ListSpidersResponse ListSpiders(string project, string version = "")
        {
            if (string.IsNullOrEmpty(project)) throw new ArgumentNullException(nameof(project));

            try
            {
                var client = new HttpClient
                {
                    BaseAddress = new Uri($"http://{_host}:{_port}")
                };

                var result = client.GetStringAsync($"listspiders.json?project={project}&_version={version}").Result;
                return JsonConvert.DeserializeObject<ListSpidersResponse>(result);
            }
            catch (Exception e)
            {
                return new ListSpidersResponse { status = "failed", message = e.Message };
            }
        }

        public ListJobsResponse ListJobs(string project)
        {
            if (string.IsNullOrEmpty(project)) throw new ArgumentNullException(nameof(project));

            try
            {
                var client = new HttpClient
                {
                    BaseAddress = new Uri($"http://{_host}:{_port}")
                };

                var result = client.GetStringAsync($"listjobs.json?project={project}").Result;
                return JsonConvert.DeserializeObject<ListJobsResponse>(result);
            }
            catch (Exception e)
            {
                return new ListJobsResponse { status = "failed", message = e.Message };
            }
        }

        public ScrapydResponse DeleteVersion(string project, string version)
        {
            if (string.IsNullOrEmpty(project)) throw new ArgumentNullException(nameof(project));
            if (string.IsNullOrEmpty(version)) throw new ArgumentNullException(nameof(version));

            var client = new HttpClient()
            {
                BaseAddress = new Uri($"http://{_host}:{_port}")
            };

            try
            {
                var data = new Dictionary<string, string>()
                {
                    {"project", project},
                    {"version", version}
                };

                var strResult =
                    client.PostAsync("delversion.json", new FormUrlEncodedContent(data))
                        .Result.Content.ReadAsStringAsync()
                        .Result;

                return JsonConvert.DeserializeObject<ScrapydResponse>(strResult);

            }
            catch (Exception e)
            {
                return new ScrapydResponse { status = "failed", message = e.Message };
            }
        }

        public ScrapydResponse DeleteProject(string project)
        {
            if (string.IsNullOrEmpty(project)) throw new ArgumentNullException(nameof(project));


            var client = new HttpClient()
            {
                BaseAddress = new Uri($"http://{_host}:{_port}")
            };

            try
            {
                var data = new Dictionary<string, string>()
                {
                    {"project", project}
                };

                var strResult =
                    client.PostAsync("delproject.json", new FormUrlEncodedContent(data))
                        .Result.Content.ReadAsStringAsync()
                        .Result;

                return JsonConvert.DeserializeObject<ScrapydResponse>(strResult);

            }
            catch (Exception e)
            {
                return new ScrapydResponse { status = "failed", message = e.Message };
            }
        }

    }

}

博客地址 : http://www.cnblogs.com/mobwiz/