一般情况下,如果无特殊的需要,对于抓去网页数据,可以使用WebClient,也可以使用webrequest/webresponse这对组合,不过我使用webrequest/webresponse的时候比较多一些,所以,自己整理了一个网页抓取类,上代码:

Code
1
/**//// <summary>
2
/// Web下载工具
3
/// </summary>
4
public class WebDownload : IDisposable
5
{
6
事件#region 事件
7
8
/**//// <summary>
9
/// 下载完成事件
10
/// </summary>
11
//public event EventHandler DownloadComplete;
12
13
#endregion
14
15
字段#region 字段
16
/**//// <summary>
17
/// 请求对象
18
/// </summary>
19
private HttpWebRequest m_request;
20
21
/**//// <summary>
22
/// 请求返回对象
23
/// </summary>
24
private HttpWebResponse m_response;
25
26
/**//// <summary>
27
/// 请求URL
28
/// </summary>
29
public string Url = string.Empty;
30
31
/**//// <summary>
32
/// Method
33
/// </summary>
34
public string Method = string.Empty;
35
36
/**//// <summary>
37
/// 数据类型
38
/// </summary>
39
public string ContentType = string.Empty;
40
41
/**//// <summary>
42
/// referer
43
/// </summary>
44
public string Referer = string.Empty;
45
46
/**//// <summary>
47
/// 是否允许重定向
48
/// </summary>
49
public bool AllowAutoRedirect = true;
50
51
/**//// <summary>
52
/// accept
53
/// </summary>
54
public string Accept = string.Empty;
55
56
/**//// <summary>
57
/// 提交的数据
58
/// </summary>
59
public string PostData = string.Empty;
60
61
/**//// <summary>
62
/// cookies
63
/// </summary>
64
public CookieContainer Cookies = null;
65
66
/**//// <summary>
67
/// 连接超时时间
68
/// </summary>
69
public int Timeout = 100000;
70
71
/**//// <summary>
72
/// 编码类型
73
/// </summary>
74
public Encoding Encode = Encoding.Default;
75
76
/**//// <summary>
77
/// useragent
78
/// </summary>
79
public string UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 2.0.50727)";
80
81
/**//// <summary>
82
/// 请求返回的数据流
83
/// </summary>
84
//public Stream ReceiveData = null;
85
#endregion
86
87
构造函数#region 构造函数
88
/**//// <summary>
89
/// 下载工具
90
/// </summary>
91
/// <param name="url"></param>
92
/// <param name="method"></param>
93
public WebDownload(string url, string method) : this( url,method,string.Empty )
94
{
95
}
96
97
/**//// <summary>
98
/// 下载工具
99
/// </summary>
100
/// <param name="url"></param>
101
/// <param name="method"></param>
102
/// <param name="postData"></param>
103
public WebDownload(string url, string method, string postData) : this( url,method,postData,"application/x-www-form-urlencoded" )
104
{
105
}
106
107
/**//// <summary>
108
/// 下载工具
109
/// </summary>
110
/// <param name="url"></param>
111
/// <param name="method"></param>
112
/// <param name="postData"></param>
113
/// <param name="contentType"></param>
114
public WebDownload(string url, string method, string postData, string contentType) : this( url,method,postData,contentType,"image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, application/x-shockwave-flash, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, application/x-silverlight, */*" )
115
{
116
}
117
118
/**//// <summary>
119
/// 下载工具
120
/// </summary>
121
/// <param name="url"></param>
122
/// <param name="method"></param>
123
/// <param name="postData"></param>
124
/// <param name="contentType"></param>
125
/// <param name="accept"></param>
126
public WebDownload(string url, string method, string postData, string contentType, string accept) : this( url,method,postData,contentType,accept,string.Empty )
127
{
128
}
129
130
/**//// <summary>
131
/// 下载工具
132
/// </summary>
133
/// <param name="url"></param>
134
/// <param name="method"></param>
135
/// <param name="postData"></param>
136
/// <param name="contentType"></param>
137
/// <param name="accept"></param>
138
/// <param name="referer"></param>
139
public WebDownload(string url, string method, string postData, string contentType, string accept,string referer) : this( url,method,postData,contentType,accept,referer,true )
140
{
141
}
142
143
/**//// <summary>
144
/// 下载工具
145
/// </summary>
146
/// <param name="url"></param>
147
/// <param name="method"></param>
148
/// <param name="postData"></param>
149
/// <param name="contentType"></param>
150
/// <param name="accept"></param>
151
/// <param name="referer"></param>
152
/// <param name="autoredirect"></param>
153
public WebDownload(string url, string method, string postData, string contentType, string accept, string referer,bool autoredirect)
154
{
155
Url = url;
156
Method = method;
157
PostData = postData;
158
ContentType = contentType;
159
Accept = accept;
160
Referer = referer;
161
AllowAutoRedirect = autoredirect;
162
}
163
#endregion
164
165
方法#region 方法
166
167
/**//// <summary>
168
/// 下载数据
169
/// </summary>
170
public Stream Download()
171
{
172
m_request = (HttpWebRequest)WebRequest.Create(Url);
173
174
//创建cookies容器
175
CookieContainer cc = null;
176
if (Cookies != null)
177
cc = Cookies;
178
else
179
cc = new CookieContainer();
180
m_request.ContentType = ContentType;
181
m_request.UserAgent = UserAgent;
182
m_request.Referer = Referer;
183
m_request.AllowAutoRedirect = AllowAutoRedirect;
184
m_request.Accept = Accept;
185
m_request.Timeout = Timeout;
186
m_request.Method = Method;
187
188
//检查是否有提交数据,只有post才可提交数据
189
if (PostData.Length > 0 && Method == "POST")
190
{
191
byte[] bytes = Encode.GetBytes(PostData);
192
Stream postStream = m_request.GetRequestStream();
193
postStream.Write(bytes, 0, bytes.Length);
194
postStream.Close();
195
}
196
197
try
198
{
199
m_response = (HttpWebResponse)m_request.GetResponse();
200
}
201
catch (WebException ex)
202
{
203
if (ex.Response != null)
204
{
205
m_response = (HttpWebResponse)ex.Response;
206
}
207
else
208
{
209
throw ex;
210
}
211
}
212
return m_response.GetResponseStream();
213
//if (DownloadComplete != null)
214
//{
215
// DownloadComplete(this, EventArgs.Empty);
216
//}
217
}
218
219
#endregion
220
221
IDisposable 成员#region IDisposable 成员
222
223
public void Dispose()
224
{
225
if (m_response != null)
226
m_response.Close();
227
}
228
229
#endregion
230
}