c# 爬虫常用的3个方法，备份一下
1 using System; 2 using System.Collections.Generic; 3 using System.IO; 4 using System.Linq; 5 using System.Net; 6 using System.Text; 7 8 namespace MiSuMi 9 { 10 public class HttpHelper 11 { 12 public CookieContainer cookie; 13 public HttpHelper() 14 { 15 cookie = new CookieContainer(); 16 } 17 18 public string Get_Request( 19 string strUrl, 20 CookieContainer _cookie = null, 21 string strHost = "", 22 string strRefer = "", 23 string strOrigin = "", 24 Dictionary<string, string> lstHeads = null, 25 string strEncoding = "utf-8", 26 string strContentType = "", 27 string strAccept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", 28 string strUserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36", 29 bool blnAllowAutoRedirect = true, 30 int intTimeout = 1000 * 30) 31 { 32 HttpWebRequest request; 33 HttpWebResponse response; 34 request = (HttpWebRequest)WebRequest.Create(strUrl); 35 request.Accept = strAccept; 36 request.Timeout = intTimeout; 37 request.Method = "GET"; 38 request.Credentials = CredentialCache.DefaultCredentials; 39 request.UserAgent = strUserAgent; 40 request.AllowAutoRedirect = blnAllowAutoRedirect; 41 if (!string.IsNullOrEmpty(strContentType)) 42 { 43 request.ContentType = strContentType; 44 } 45 if (_cookie != null) 46 { 47 request.CookieContainer = _cookie; 48 } 49 if (!string.IsNullOrEmpty(strHost)) 50 { 51 request.Host = strHost; 52 } 53 if (!string.IsNullOrEmpty(strRefer)) 54 { 55 request.Referer = strRefer; 56 } 57 if (!string.IsNullOrEmpty(strOrigin)) 58 { 59 request.Headers.Add("Origin", strOrigin); 60 } 61 if (lstHeads != null && lstHeads.Count > 0) 62 { 63 foreach (var item in lstHeads) 64 { 65 request.Headers.Add(item.Key, item.Value); 66 } 67 } 68 response = (HttpWebResponse)request.GetResponse(); 69 var sr = new StreamReader(response.GetResponseStream(), Encoding.GetEncoding(strEncoding)); 70 string strResult = sr.ReadToEnd(); 71 sr.Close(); 72 request.Abort(); 73 response.Close(); 74 return strResult; 75 76 } 77 78 public string POST_Request( 79 string strUrl, 80 string postDataStr, 81 CookieContainer _cookie = null, 82 string strHost = "", 83 string strRefer = "", 84 string strOrigin = "", 85 Dictionary<string, string> lstHeads = null, 86 string strEncoding = "utf-8", 87 string strContentType = "", 88 string strAccept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", 89 string strUserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36", 90 bool blnAllowAutoRedirect = true, 91 int intTimeout = 1000 * 30) 92 { 93 HttpWebRequest request; 94 HttpWebResponse response; 95 request = (HttpWebRequest)WebRequest.Create(strUrl); 96 request.Accept = strAccept; 97 request.Timeout = intTimeout; 98 request.Method = "POST"; 99 request.Host = strHost; 100 request.UserAgent = strUserAgent; 101 if (_cookie != null) 102 { 103 request.CookieContainer = _cookie; 104 } 105 request.AllowAutoRedirect = blnAllowAutoRedirect; 106 if (!string.IsNullOrEmpty(strContentType)) 107 { 108 request.ContentType = strContentType; 109 } 110 if (!string.IsNullOrEmpty(strOrigin)) 111 { 112 request.Headers.Add("Origin", strOrigin); 113 } 114 if (!string.IsNullOrEmpty(strRefer)) 115 { 116 request.Referer = strRefer; 117 } 118 if (!string.IsNullOrEmpty(strHost)) 119 { 120 request.Host = strHost; 121 } 122 if (lstHeads != null && lstHeads.Count > 0) 123 { 124 foreach (var item in lstHeads) 125 { 126 request.Headers.Add(item.Key, item.Value); 127 } 128 } 129 if (!string.IsNullOrEmpty(postDataStr)) 130 { 131 request.ContentLength = postDataStr.Length; 132 Stream myRequestStream = request.GetRequestStream(); 133 StreamWriter myStreamWriter = new StreamWriter(myRequestStream); 134 myStreamWriter.Write(postDataStr); 135 myStreamWriter.Close(); 136 } 137 138 response = (HttpWebResponse)request.GetResponse(); 139 var sr = new StreamReader(response.GetResponseStream(), Encoding.GetEncoding(strEncoding)); 140 string strResult = sr.ReadToEnd(); 141 sr.Close(); 142 request.Abort(); 143 response.Close(); 144 return strResult; 145 } 146 147 private string DownloadFile( 148 string strURLAddress, 149 string strPath, 150 CookieContainer _cookie = null, 151 string strHost = "", 152 string strRefer = "", 153 string strOrigin = "", 154 Dictionary<string, string> lstHeads = null, 155 string strAccept = "", 156 string strUserAgent = "") 157 { 158 try 159 { 160 // 设置参数 161 HttpWebRequest request = WebRequest.Create(strURLAddress) as HttpWebRequest; 162 if (!string.IsNullOrEmpty(strAccept)) 163 { 164 request.Accept = strAccept; 165 } 166 if (!string.IsNullOrEmpty(strUserAgent)) 167 { 168 request.UserAgent = strUserAgent; 169 } 170 if (_cookie != null) 171 { 172 request.CookieContainer = _cookie; 173 } 174 if (!string.IsNullOrEmpty(strOrigin)) 175 { 176 request.Headers.Add("Origin", strOrigin); 177 } 178 if (!string.IsNullOrEmpty(strRefer)) 179 { 180 request.Referer = strRefer; 181 } 182 if (!string.IsNullOrEmpty(strHost)) 183 { 184 request.Host = strHost; 185 } 186 if (lstHeads != null && lstHeads.Count > 0) 187 { 188 foreach (var item in lstHeads) 189 { 190 request.Headers.Add(item.Key, item.Value); 191 } 192 } 193 //发送请求并获取相应回应数据 194 HttpWebResponse response = request.GetResponse() as HttpWebResponse; 195 string strReceivePath = string.Empty; 196 197 //直到request.GetResponse()程序才开始向目标网页发送Post请求 198 Stream responseStream = response.GetResponseStream(); 199 //创建本地文件写入流 200 Stream stream = new FileStream(strPath, FileMode.Create); 201 byte[] bArr = new byte[1024]; 202 int size = responseStream.Read(bArr, 0, (int)bArr.Length); 203 while (size > 0) 204 { 205 stream.Write(bArr, 0, size); 206 stream.Flush(); 207 size = responseStream.Read(bArr, 0, (int)bArr.Length); 208 } 209 stream.Close(); 210 responseStream.Close(); 211 return strPath; 212 } 213 catch (Exception ex) 214 { 215 return ""; 216 } 217 } 218 219 } 220 }
常用的方法，参数就这些了，后面如果有用到其他的再补充了