多线程下载图片

TransferWebFileHelper1

public static class TransferWebFileHelper1
    {
        /// <summary>
        /// 异步网络请求
        /// </summary>
        public static async Task CommunicatingWithWebServerAsync(string url,string saveFilePath, bool isGet = true, string postData = "", string contentType = "application/x-www-form-urlencoded;charset=utf-8")
        {
            if (string.IsNullOrWhiteSpace(url))
                throw new ArgumentNullException("url");
            if (string.IsNullOrWhiteSpace(contentType))
                throw new ArgumentNullException("contentType");
            HttpWebResponse response = null;
            Stream stream = Stream.Null;
            FileStream outStream = null;
            try
            {
                if (url.ToUpper().Contains("HTTPS:")) //解决：“访问https请求被中止: 未能创建 SSL/TLS 安全通道”
                {
                    ServicePointManager.SecurityProtocol = SecurityProtocolType.Ssl3 | SecurityProtocolType.Tls |
                                                           SecurityProtocolType.Tls11 | SecurityProtocolType.Tls12;
                }
                HttpWebRequest request = isGet
                    ? GenerateHttpWebRequest(new Uri(url))
                    : GenerateHttpWebRequest(new Uri(url), postData, contentType);
                response = await request.GetResponseAsync() as HttpWebResponse;
                if (CategorizeResponse(response) == ResponseCategories.Success)
                {
                    stream = response.GetResponseStream();

                    outStream = new FileStream(saveFilePath, FileMode.Create);
                    int readByteCount = 0;
                    byte[] buffers = new byte[1024];
                    while (true)
                    {
                        readByteCount = stream.Read(buffers, 0, buffers.Length);
                        if (readByteCount <= 0)
                        {
                            break;
                        }
                        outStream.Write(buffers, 0, readByteCount);
                    }

                    stream.Close();
                    stream.Dispose();
                    outStream.Close();
                    outStream.Dispose();
                    response.Close();
                    response.Dispose();
                }


            }
            catch (Exception ex)
            {

            }
            finally
            {
                System.GC.Collect();//解决程序提示线程用尽崩溃的问题
            }
        }

        public static HttpWebRequest GenerateHttpWebRequest(Uri uri)
        {
            HttpWebRequest httpRequest = (HttpWebRequest)WebRequest.Create(uri);
            httpRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36";//部分网站会检查此项
            return httpRequest;
        }

        public static HttpWebRequest GenerateHttpWebRequest(Uri uri,
            string postData,
            string contentType)
        {
            HttpWebRequest httpRequest = GenerateHttpWebRequest(uri);

            byte[] bytes = Encoding.UTF8.GetBytes(postData);

            httpRequest.ContentType = contentType;

            httpRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1) ; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022)";           

            httpRequest.ContentLength = postData.Length;

            using (Stream requestStream = httpRequest.GetRequestStream())
            {
                requestStream.Write(bytes, 0, bytes.Length);
            }
            return httpRequest;
        }

        public static ResponseCategories CategorizeResponse(HttpWebResponse httpResponse)
        {
            int statusCode = (int)httpResponse.StatusCode;
            if ((statusCode >= 100) && (statusCode <= 199))
            {
                return ResponseCategories.Informational;
            }
            else if ((statusCode >= 200) && (statusCode <= 299))
            {
                return ResponseCategories.Success;
            }
            else if ((statusCode >= 300) && (statusCode <= 399))
            {
                return ResponseCategories.Redirected;
            }
            else if ((statusCode >= 400) && (statusCode <= 499))
            {
                return ResponseCategories.ClientError;
            }
            else if ((statusCode >= 500) && (statusCode <= 599))
            {
                return ResponseCategories.ServerError;
            }
            return ResponseCategories.Unknown;
        }
    }

Enumerator

    public class Enumerator
    {
        public enum HttpMethodEnum
        {
            Post,
            Get,
            Put,
            Delete,
            Patch
        }
    }

ResponseCategories

    public enum ResponseCategories
    {
        Unknown,        
        Informational, 
        Success,       
        Redirected,     
        ClientError,    
        ServerError
    }

Program

class Program
    {
        private static string[] randomArray = new[]
        {
            "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V",
            "W", "X", "Y", "Z"
        };
        private static string GetRandomStr()
        {
            Random ran = new Random();
            int i = ran.Next(0, 26);
            Thread.Sleep(1);//尽量让生成的字符每一个都不相同
            return randomArray[i];
        }

        static List<DownloadFile> ReadFileUrl(string folderName)
        {
            List<DownloadFile> list = new List<DownloadFile>();
            DirectoryInfo dir;
            List<string> contents = new List<string>();
            string path = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, folderName);
            if (Directory.Exists(path))
            {
                dir = new DirectoryInfo(path);
                FileInfo[] files = dir.GetFiles();
                contents = File.ReadAllLines(files[0].FullName, Encoding.Default).ToList();

                string saveFolder = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, folderName + "_download");

                contents.ForEach(fileName =>
                {
                    //TODO 完全限定文件名必须少于 260 个字符，并且目录名必须少于 248 个字符。
                    string tempFileName = fileName.Split('?')[0].Replace("\\", "").Replace(":", "").Replace("*", "").Replace("\"", "").Replace("<", "").Replace(">", "").Replace("|", "");
                    int index = tempFileName.LastIndexOf("/");
                    tempFileName = tempFileName.Substring(index + 1, tempFileName.Length - index - 1);
                    int tempFileNameLength = tempFileName.Length;
                    if ((260 - saveFolder.Length - 2) < tempFileNameLength)
                    {
                        int difference = tempFileNameLength - (260 - saveFolder.Length - 2);//差
                        tempFileName = tempFileName.Substring(difference, tempFileName.Length - difference);
                    }
                    tempFileName = GetRandomStr() + "_" + tempFileName;


                    string saveFileName = Path.Combine(saveFolder, tempFileName);
                    var model = new DownloadFile(fileName, folderName, saveFileName);
                    list.Add(model);
                });
            }
            return list;
        }

        private static void CreateFolder(string folderName)
        {
            string saveFolder = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, folderName + "_download");
            if (!Directory.Exists(saveFolder))
            {
                Directory.CreateDirectory(saveFolder);
            }
        }

        private static void Main(string[] args)
        {
            List<string> folders = new List<string>()
            {
                "demo"
            };
            Parallel.ForEach(folders, folder => CreateFolder(folder));

            List<DownloadFile> downloadFiles = new List<DownloadFile>();

            Parallel.ForEach(folders, folder =>
            {
                downloadFiles.AddRange(ReadFileUrl(folder));
            });

            int i = 0;
            Task[] taskList = new Task[downloadFiles.Count];
            downloadFiles.ForEach(downloadFile =>
            {
                try
                {
                    Task t = TransferWebFileHelper1.CommunicatingWithWebServerAsync(downloadFile.FileName,downloadFile.SaveFileName);
                    taskList[i] = t;

                }
                catch (Exception ex)
                {

                }
                i++;
            });
            Task.WaitAll(taskList);
        }
    }

测试

程序根目录下，新建一个文件夹，名demo。内新建一个txt文件，文件名随意。测试如下图片链接[随意]：

http://www.banktunnel.eu/tumblr.com/tabea-lara.2_files/tumblr_n4kxopXkJ41sq93cpo10_1280.jpg
http://www.pptok.com/wp-content/uploads/2012/08/xunguang-9.jpg
http://www.xiangshu.com/UploadFiles/Day_190324/52_387785_3163ee0990499b7.jpg
http://www.xiangshu.com/UploadFiles/Day_190324/52_387785_25d8b5e402d0391.jpg
https://bbs-fd.zol-img.com.cn/t_s800x5000/g4/M04/0A/0D/Cg-4WVPNFNWIA9vCAAmNhS2C8eAAAPyvQApsb4ACY2d098.jpg
https://bbs-fd.zol-img.com.cn/t_s800x5000/g4/M04/0A/0D/Cg-4WlPNFOGIIc0VAAom_BUlna4AAPyvQCLDjYACicU104.png
https://bbs-fd.zol-img.com.cn/t_s800x5000/g4/M04/0A/0D/Cg-4WlPNFOqIWH2cAAXWIl1DvA4AAPyvQDbhzkABdY6485.jpg
https://bbs-fd.zol-img.com.cn/t_s800x5000/g4/M04/0A/0D/Cg-4WVPNFPGIfiurAAKxaF37_vcAAPyvQGII2YAArGA075.jpg
http://www.hr.com.cn/attachments/2015/06/b16af76b691f47609f3397e8cdd9c1e2.jpg
http://www.hr.com.cn/attachments/2015/06/226fc8acee0d478c9588ded1d04616b1.jpg
http://www.hr.com.cn/attachments/2015/06/c36e39b67d4c410cb794f553a271377b.jpg

下载完成的图片存放在根目录下名demo_download的文件夹下

测试过支持10W左右的图片下载，不崩溃。