[转]HttpWebRequest提高效率之连接数,代理，自动跳转,gzip请求等设置有关问题

[转]HttpWebRequest提高效率之连接数,代理，自动跳转,gzip请求等设置有关问题

www.MyException.Cn   发布于：2012-09-15 19:09:29   浏览：7次

 

[记录]HttpWebRequest提高效率之连接数,代理，自动跳转,gzip请求等设置问题

先设置4个：

               

 webrequest.ServicePoint.Expect100Continue = false;
                //是否使用 Nagle 不使用 提高效率
                webrequest.ServicePoint.UseNagleAlgorithm = false;
                //最大连接数
                webrequest.ServicePoint.ConnectionLimit = 65500;
                //数据是否缓冲 false 提高效率 
                webrequest.AllowWriteStreamBuffering = false;

 

代理的使用：我用的代理都是不要账号和密码的，因为要不是免费的，要是是我们自己的，对来访IP 有限制

            

 if (proxy!=null)
                {
                    model.proxy = proxy;
                    WebProxy myProxy = new WebProxy(proxy.ProxyInfo, false);
                    myProxy.Credentials = new NetworkCredential("", "");
                    webrequest.Proxy = myProxy;
                }
                else
                {
                    webrequest.Proxy = GlobalProxySelection.GetEmptyWebProxy();
                }

webrequest.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip,deflate");

带上gzip，如果对方支持gzip，流量上可以节省五分之四，原来250KB，用gzip压缩有 大概在50KB，减少带宽压力，增加速度

值得注意的是，如果是gzip，返回值必须要gzip解压的。

 if (response.ContentEncoding.ToLower().Contains("gzip"))
                    {
                        using (GZipStream stream = new GZipStream(response.GetResponseStream(), CompressionMode.Decompress))
                        {
                            using (StreamReader reader = new StreamReader(stream, Encoding.UTF8))
                            {

                                model.html = reader.ReadToEnd();
                            }
                        }
                    }

还有一点：gzip解压的时候回耗费CPU，所以这个需要注意下

附完成方法

 /// <summary>
        /// Get 方式 获取数据
        /// </summary>
        /// <param name="webUrl"></param>
        /// <returns></returns>
        public Model.WebRequestReturnModel WebRequestGetHtmlByGet(string webUrl, CookieContainer cookieContainer, Encoding encoding, string refer,Model.Proxy proxy)
        {
            Model.WebRequestReturnModel model = new Model.WebRequestReturnModel();
            HttpWebRequest webrequest = null;
            try
            {
                webrequest = WebRequest.Create(webUrl) as HttpWebRequest;
                
                webrequest.ServicePoint.Expect100Continue = false;
                 webrequest.ServicePoint.UseNagleAlgorithm = false;
                 webrequest.ServicePoint.ConnectionLimit = 65500;
                webrequest.AllowWriteStreamBuffering = false;
                if (proxy!=null)
                {
                    model.proxy = proxy;
                    WebProxy myProxy = new WebProxy(proxy.ProxyInfo, false);
                    myProxy.Credentials = new NetworkCredential("", "");
                    webrequest.Proxy = myProxy;
                }
                else
                {
                    webrequest.Proxy = GlobalProxySelection.GetEmptyWebProxy();
                }
                //设置其他的herader
                webrequest =WebRequestSetHeaders(webrequest);
                if (!String.IsNullOrEmpty(refer))
                {
                    webrequest.Referer = refer;
                }
                else
                {
                    webrequest.Referer = webUrl;
                }
                webrequest.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip,deflate");
                webrequest.CookieContainer = cookieContainer;
                webrequest.Timeout = 5000;
                webrequest.UserAgent = "User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705;)";
                webrequest.Accept = "*/*";
                webrequest.KeepAlive = true;
                webrequest.Headers.Add("Accept-Language", "zh-cn,en-us;q=0.5");

                using (HttpWebResponse response = (HttpWebResponse)webrequest.GetResponse())
                {

                    if (response.ContentEncoding.ToLower().Contains("gzip"))
                    {
                        using (GZipStream stream = new GZipStream(response.GetResponseStream(), CompressionMode.Decompress))
                        {
                            using (StreamReader reader = new StreamReader(stream, Encoding.UTF8))
                            {

                                model.html = reader.ReadToEnd();
                            }
                        }
                    }
                    else if (response.ContentEncoding.ToLower().Contains("deflate"))
                    {
                        using (DeflateStream stream = new DeflateStream(response.GetResponseStream(), CompressionMode.Decompress))
                        {
                            using (StreamReader reader = new StreamReader(stream, Encoding.UTF8))
                            {

                                model.html = reader.ReadToEnd();
                            }

                        }
                    }
                    else
                    {
                        using (Stream stream = response.GetResponseStream())
                        {
                            using (StreamReader reader = new StreamReader(stream, Encoding.UTF8))
                            {

                                model.html = reader.ReadToEnd();
                            }
                        }
                    }
                }
            }
            catch (Exception ex)
            {
                model.exception = ex;
                model.html = string.Empty;

            }
            model.cookieContainer = cookieContainer;
            return model;
        }