Net WebClient 异步批量下载文件

1.文件提前使用爬虫整理好，放到了demo文件夹下的1.txt文件中，文件格式如：

http://www.banktunnel.eu/tumblr.com/tabea-lara.2_files/tumblr_n4kxopXkJ41sq93cpo10_1280.jpg
http://www.pptok.com/wp-content/uploads/2012/08/xunguang-9.jpg
http://www.xiangshu.com/UploadFiles/Day_190324/52_387785_3163ee0990499b7.jpg
http://www.xiangshu.com/UploadFiles/Day_190324/52_387785_25d8b5e402d0391.jpg
https://bbs-fd.zol-img.com.cn/t_s800x5000/g4/M04/0A/0D/Cg-4WVPNFNWIA9vCAAmNhS2C8eAAAPyvQApsb4ACY2d098.jpg
https://bbs-fd.zol-img.com.cn/t_s800x5000/g4/M04/0A/0D/Cg-4WlPNFOGIIc0VAAom_BUlna4AAPyvQCLDjYACicU104.png
https://bbs-fd.zol-img.com.cn/t_s800x5000/g4/M04/0A/0D/Cg-4WlPNFOqIWH2cAAXWIl1DvA4AAPyvQDbhzkABdY6485.jpg
https://bbs-fd.zol-img.com.cn/t_s800x5000/g4/M04/0A/0D/Cg-4WVPNFPGIfiurAAKxaF37_vcAAPyvQGII2YAArGA075.jpg
http://www.hr.com.cn/attachments/2015/06/b16af76b691f47609f3397e8cdd9c1e2.jpg
http://www.hr.com.cn/attachments/2015/06/226fc8acee0d478c9588ded1d04616b1.jpg
http://www.hr.com.cn/attachments/2015/06/c36e39b67d4c410cb794f553a271377b.jpg

2.新建DownloadFile类

public class DownloadFile
    {
        public DownloadFile()
        {           
        }
        public DownloadFile(string fileName, string fileFolder,string saveFileName)
        {
            FileName = fileName;
            FileFolder = fileFolder;
            SaveFileName = saveFileName;
        }      
        public string FileName;
        public string FileFolder;
        public string SaveFileName;
    }

3.Main函数创建下载用文件夹，读取需下载文件名称到List<DownloadFile>中。调用下载方法执行下载。

        private static void Main(string[] args)
        {
            List<string> folders = new List<string>()
            {
                "demo","其他文件夹","其他文件夹1","xxxx"
            };
            Parallel.ForEach(folders, folder => CreateFolder(folder));
            List<DownloadFile> downloadFiles = new List<DownloadFile>();
            Parallel.ForEach(folders, folder =>
            {
                downloadFiles.AddRange(ReadFileUrl(folder));
            });
            Logger.Log(Level.Info, "开始任务执行");
            List<Task> tList = new List<Task>();
            downloadFiles.ForEach(p =>
            {
                tList.Add(
                    TransferWebFileHelper.DownloadingDataFromServerAsync(p)
                );
            });
            Task.WaitAll(tList.ToArray());           
            Logger.Log(Level.Info, "任务执行结束");
        }

        /// <summary>
        /// 创建下载用文件夹
        /// </summary>
        /// <param name="folderName"></param>
        private static void CreateFolder(string folderName)
        {
            string saveFolder = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, folderName + "_download");
            if (!Directory.Exists(saveFolder))
            {
                Directory.CreateDirectory(saveFolder);
            }
        }

        /// <summary>
        /// 载入待下载文件到集合
        /// </summary>
        /// <param name="folderName"></param>
        /// <returns></returns>
        static List<DownloadFile> ReadFileUrl(string folderName)
        {
            List<DownloadFile> list = new List<DownloadFile>();
            DirectoryInfo dir;
            List<string> contents = new List<string>();
            string path = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, folderName);
            if (Directory.Exists(path))
            {
                dir = new DirectoryInfo(path);
                FileInfo[] files = dir.GetFiles();
                contents = File.ReadAllLines(files[0].FullName, Encoding.Default).ToList();
                string saveFolder = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, folderName + "_download");
                contents.ForEach(fileName =>
                {
                    //TODO 完全限定文件名必须少于 260 个字符，并且目录名必须少于 248 个字符。
                    string tempFileName = fileName.Split('?')[0].Replace("\", "").Replace(":", "").Replace("*", "").Replace(""", "").Replace("<", "").Replace(">", "").Replace("|", "");
                    int index = tempFileName.LastIndexOf("/");
                    tempFileName = tempFileName.Substring(index + 1, tempFileName.Length - index - 1);
                    int tempFileNameLength = tempFileName.Length;
                    if ((260 - saveFolder.Length - 2) < tempFileNameLength)
                    {
                        int difference = tempFileNameLength - (260 - saveFolder.Length - 2);//差
                        tempFileName = tempFileName.Substring(difference, tempFileName.Length - difference);
                    }
                    tempFileName = GetRandomStr() + "_" + tempFileName;
                    string saveFileName = Path.Combine(saveFolder, tempFileName);
                    var model = new DownloadFile(fileName, folderName, saveFileName);
                    list.Add(model);
                });
            }
            return list;
        }

4.下载文件类

    public static class TransferWebFileHelper
    {
        /// <summary>
        /// 新建文件，如果它不存在的话
        /// </summary>
        /// <param name="fileName"></param>
        private static void CreateFile(string fileName)
        {
            if (!File.Exists(fileName))
            {
                FileStream fileStream = File.Create(fileName);
                fileStream.Close();
                fileStream.Dispose();
            }
        }

        /// <summary>
        /// 下载用方法
        /// </summary>
        /// <param name="downloadFile"></param>
        /// <returns></returns>
        public static async Task DownloadingDataFromServerAsync(DownloadFile downloadFile)
        {
            Uri uri = new Uri(downloadFile.FileName);
            string saveFileName = downloadFile.SaveFileName;
            CreateFile(saveFileName);

            using (WebClient client = new WebClient())
            {
                try
                {
                    await client.DownloadFileTaskAsync(uri, saveFileName);
                }
                catch (WebException ex)
                {
                    Console.WriteLine($"从 {uri} 下载 {saveFileName}失败");
                    Logger.Log(Level.Info, $"从 {uri} 下载 {saveFileName}失败");
                    Logger.LogException(ex);
                }
                catch (Exception ex)
                {
                    Console.WriteLine($"从 {uri} 下载 {saveFileName}失败");
                    Logger.Log(Level.Info, $"从 {uri} 下载 {saveFileName}失败");
                    Logger.LogException(ex);
                }
            }
        }
    }

5.其他辅助方法

        private static string[] randomArray = new[]
        {
            "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V",
            "W", "X", "Y", "Z"
        };
        private static string GetRandomStr()
        {
            Random ran = new Random();
            int i = ran.Next(0, 26);
            Thread.Sleep(1);//尽量让生成的字符每一个都不相同
            return randomArray[i];
        }