做一些抓取网页,投票等等功能,经常会碰到服务器判断IP限制IP的情况,此时一般在程序中使用代理服务器访问服务器是一种解决方式(其他方式有冒充IP,或者直接攻击服务器等等)
免费的代理服务器可以在网上搜索获得,一般外国的提供较多,国内的免费代理现在很少了。
或者到一些提供代理IP的收费网站购买,比如“万变IP”,“极光爬虫代理”等等,如果量不是很大的话,倒是也不算贵。(这些网站还会提供动态的改本机IP的软件(一般通过VPN改变))
有了代理的IP,就可以在代码中使用代理抓取或者投票了,这样不断的变换代理IP,服务器就无法识别出客户端请求IP了,就无法阻止采集与投票了。
public static string HttpGet(string url) { System.Net.ServicePointManager.DefaultConnectionLimit = 50;//这个值默认是2,根据自己的情况修改 System.Net.ServicePointManager.SecurityProtocol = System.Net.SecurityProtocolType.Tls12;//SSL/TLS安全通道 //ServicePointManager.ServerCertificateValidationCallback = new RemoteCertificateValidationCallback(CheckValidationResult); System.Text.Encoding encoding = System.Text.Encoding.UTF8; System.Net.HttpWebRequest request = (System.Net.HttpWebRequest)System.Net.WebRequest.Create(url); #region request.CachePolicy = new System.Net.Cache.HttpRequestCachePolicy(System.Net.Cache.HttpRequestCacheLevel.NoCacheNoStore);//清缓存 #region 代理 if (TP != null) { WebProxy proxyObject = new WebProxy("58.213.26.166", 36563);//str为IP地址 port为端口号 //WebProxy proxyObject = new WebProxy("210.14.132.67");//str为IP地址 port为端口号 request.Proxy = proxyObject; //设置代理 } #endregion //request.Headers.Add("X-Forwarded-For", "162.150.10.16"); //request.Headers.Add("HTTP_X_FORWARDED_FOR", "162.150.10.16"); //request.Host = "www.163.com"; #endregion request.Method = "GET"; request.Accept = "text/html, application/xhtml+xml, */*"; request.ContentType = "application/json"; System.Net.HttpWebResponse response = (System.Net.HttpWebResponse)request.GetResponse(); using (System.IO.StreamReader reader = new System.IO.StreamReader(response.GetResponseStream(), System.Text.Encoding.UTF8)) { return reader.ReadToEnd(); } }
更详细的使用
= "http://www.domain.com"; //設定要獲取的地址
HttpWebRequest hwr = (HttpWebRequest)HttpWebRequest.Create(urlStr); //建立HttpWebRequest對象
hwr.Timeout = 60000; //定義服務器超時時間
WebProxy proxy = new WebProxy(); //定義一個網關對象
proxy.Address = new Uri("http://proxy.domain.com:3128"); //網關服務器:端口
proxy.Credentials = new NetworkCredential("f3210316", "6978233"); //用戶名,密碼
hwr.UseDefaultCredentials = true; //啟用網關認証
hwr.Proxy = proxy; //設置網關
HttpWebResponse hwrs = (HttpWebResponse)hwr.GetResponse(); //取得回應
Stream s = hwrs.GetResponseStream(); //得到回應的流對象
StreamReader sr = new StreamReader(s, Encoding.UTF8); //以UTF-8編碼讀取流
StringBuilder content = new StringBuilder(); //
while (sr.Peek() != -1) //每次讀取一行,直到
{ //下一個字節沒有內容
content.Append(sr.ReadLine()+""r"n"); //返回為止
} //
return content.ToString() ; //返回得到的字符串
HttpWebRequest hwr = (HttpWebRequest)HttpWebRequest.Create(urlStr); //建立HttpWebRequest對象
hwr.Timeout = 60000; //定義服務器超時時間
WebProxy proxy = new WebProxy(); //定義一個網關對象
proxy.Address = new Uri("http://proxy.domain.com:3128"); //網關服務器:端口
proxy.Credentials = new NetworkCredential("f3210316", "6978233"); //用戶名,密碼
hwr.UseDefaultCredentials = true; //啟用網關認証
hwr.Proxy = proxy; //設置網關
HttpWebResponse hwrs = (HttpWebResponse)hwr.GetResponse(); //取得回應
Stream s = hwrs.GetResponseStream(); //得到回應的流對象
StreamReader sr = new StreamReader(s, Encoding.UTF8); //以UTF-8編碼讀取流
StringBuilder content = new StringBuilder(); //
while (sr.Peek() != -1) //每次讀取一行,直到
{ //下一個字節沒有內容
content.Append(sr.ReadLine()+""r"n"); //返回為止
} //
return content.ToString() ; //返回得到的字符串