zoukankan      html  css  js  c++  java
  • 多线程httpWebRequest

    //在初始化方法设置以下相关属性.              
                    //对象所允许的最大并发连接数//可在配置文件中设置
                    System.Net.ServicePointManager.DefaultConnectionLimit = 1000;
                    //是否使用 Nagle 不使用 提高效率 
                    System.Net.ServicePointManager.UseNagleAlgorithm = false;
                    //对象的最大空闲时间.(默认为100秒的)
                    System.Net.ServicePointManager.MaxServicePointIdleTime = 3600 * 1000;
    //-------------------------------------------------------------------------------------------
    //在 while循环中的处理..
     while (!worker.CancellationPending)
    {
                    request = WebRequest.Create(userUrl) as HttpWebRequest;
                     request.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip,deflate");
                    request.Proxy = null;//
                    request.KeepAlive = true;
                    //数据是否缓冲 false 提高效率  
                    request.AllowWriteStreamBuffering = false;
    
                     Stream stream = null;
                    if (response.ContentEncoding.ToLower().Contains("gzip"))
                    {
                        stream = new GZipStream(response.GetResponseStream(), CompressionMode.Decompress);
                    }
                    else if (response.ContentEncoding.ToLower().Contains("deflate"))
                    {
                        stream = new DeflateStream(response.GetResponseStream(), CompressionMode.Decompress);
                    }
                    else
                    {
                        stream = response.GetResponseStream();
                    }
                    using (stream)
                    {
                        using (StreamReader sr = new StreamReader(stream, true))
                        {
                            string resultString = sr.ReadToEnd();
                        }
    
    //-------------------------------------------------------------------------------------------
    
    //使用完后要进行释放,否则易造成网络堵塞
                    if (request != null)
                    {
                        request.Abort();
                        request = null;
                    }
                    if (response != null)
                    {
                        response.Close();
                        response = null;
                    }
    }
    
    //在适当时间进行 强制垃圾回收.
    System.GC.Collect();
    
    //-------------------------------------------------------------------------------------------
    
    //以上是总结的关于多线程使用 HttpWebRequest 用于提高效率获取网页源代码的 相关设置. (省略了其他代码..)
  • 相关阅读:
    Hadoop下面WordCount运行详解
    ubuntu下hadoop环境配置
    ubuntu下的jdk安装
    ASP.NET MVC4中用 BundleCollection使用问题手记
    Lab6: Paxos
    java命令行操作
    Mesos 入门教程
    Docker background
    找实习的日子
    九度 1557:和谐答案 (LIS 变形)
  • 原文地址:https://www.cnblogs.com/rexfieBlogs/p/6020284.html
Copyright © 2011-2022 走看看