httpClient多线程请求

使用httpClient可模拟请求Url获取资源，使用单线程的请求速度上会有一定的限制，参考了Apache给出的例子，自己做了测试实现多线程并发请求，以下代码需要HttpClient 4.2的包，可以在http://hc.apache.org/downloads.cgi下载

1、并发请求

  1 package generate.httpclient;
  2 
  3 import java.util.List;
  4 import java.util.concurrent.ExecutorService;
  5 import java.util.concurrent.Executors;
  6 
  7 import org.apache.http.HttpEntity;
  8 import org.apache.http.HttpResponse;
  9 import org.apache.http.client.HttpClient;
 10 import org.apache.http.client.methods.HttpGet;
 11 import org.apache.http.conn.ClientConnectionManager;
 12 import org.apache.http.conn.params.ConnManagerParams;
 13 import org.apache.http.conn.scheme.PlainSocketFactory;
 14 import org.apache.http.conn.scheme.Scheme;
 15 import org.apache.http.conn.scheme.SchemeRegistry;
 16 import org.apache.http.impl.client.DefaultHttpClient;
 17 import org.apache.http.impl.conn.PoolingClientConnectionManager;
 18 import org.apache.http.params.BasicHttpParams;
 19 import org.apache.http.params.HttpConnectionParams;
 20 import org.apache.http.params.HttpParams;
 21 import org.apache.http.protocol.BasicHttpContext;
 22 import org.apache.http.protocol.HttpContext;
 23 import org.apache.http.util.EntityUtils;
 24 
 25 public class ThreadPoolHttpClient {
 26     // 线程池
 27     private ExecutorService exe = null;
 28     // 线程池的容量
 29     private static final int POOL_SIZE = 20;
 30     private HttpClient client = null;
 31     String[] urls=null;
 32     public ThreadPoolHttpClient(String[] urls){
 33         this.urls=urls;
 34     }
 35     public void test() throws Exception {
 36         exe = Executors.newFixedThreadPool(POOL_SIZE);
 37         HttpParams params =new BasicHttpParams();
 38         /* 从连接池中取连接的超时时间 */ 
 39         ConnManagerParams.setTimeout(params, 1000);
 40         /* 连接超时 */ 
 41         HttpConnectionParams.setConnectionTimeout(params, 2000); 
 42         /* 请求超时 */
 43         HttpConnectionParams.setSoTimeout(params, 4000);
 44         SchemeRegistry schemeRegistry = new SchemeRegistry();
 45         schemeRegistry.register(
 46                 new Scheme("http", 80, PlainSocketFactory.getSocketFactory()));
 47 
 48         //ClientConnectionManager cm = new PoolingClientConnectionManager(schemeRegistry);
 49         PoolingClientConnectionManager cm=new PoolingClientConnectionManager(schemeRegistry);
 50         cm.setMaxTotal(10);
 51         final HttpClient httpClient = new DefaultHttpClient(cm,params);
 52 
 53         // URIs to perform GETs on
 54         final String[] urisToGet = urls;
 55         /* 有多少url创建多少线程，url多时机子撑不住
 56         // create a thread for each URI
 57         GetThread[] threads = new GetThread[urisToGet.length];
 58         for (int i = 0; i < threads.length; i++) {
 59             HttpGet httpget = new HttpGet(urisToGet[i]);
 60             threads[i] = new GetThread(httpClient, httpget);            
 61         }
 62         // start the threads
 63         for (int j = 0; j < threads.length; j++) {
 64             threads[j].start();
 65         }
 66 
 67         // join the threads，等待所有请求完成
 68         for (int j = 0; j < threads.length; j++) {
 69             threads[j].join();
 70         }
 71         使用线程池*/
 72         for (int i = 0; i < urisToGet.length; i++) {
 73             final int j=i;
 74             System.out.println(j);
 75             HttpGet httpget = new HttpGet(urisToGet[i]);
 76             exe.execute( new GetThread(httpClient, httpget));
 77         }
 78         
 79         
 80         //创建线程池，每次调用POOL_SIZE
 81         /*
 82         for (int i = 0; i < urisToGet.length; i++) {
 83             final int j=i;
 84             System.out.println(j);
 85             exe.execute(new Thread() {
 86                 @Override
 87                 public void run() {
 88                     this.setName("threadsPoolClient"+j);
 89                     
 90                         try {
 91                             this.sleep(100);
 92                             System.out.println(j);
 93                         } catch (InterruptedException e) {
 94                             // TODO Auto-generated catch block
 95                             e.printStackTrace();
 96                         }
 97                         
 98                         HttpGet httpget = new HttpGet(urisToGet[j]);
 99                         new GetThread(httpClient, httpget).get();
100                     }
101                     
102                     
103                 
104             });
105         }
106         
107         */
108         //exe.shutdown();
109         System.out.println("Done");
110     }
111     static class GetThread extends Thread{
112         
113         private final HttpClient httpClient;
114         private final HttpContext context;
115         private final HttpGet httpget;
116         
117         public GetThread(HttpClient httpClient, HttpGet httpget) {
118             this.httpClient = httpClient;
119             this.context = new BasicHttpContext();
120             this.httpget = httpget;
121         }
122         @Override
123         public void run(){
124             this.setName("threadsPoolClient");
125             try {
126                 Thread.sleep(5000);
127             } catch (InterruptedException e) {
128                 // TODO Auto-generated catch block
129                 e.printStackTrace();
130             }
131             get();
132         }
133         
134         public void get() {
135             try {
136                 HttpResponse response = this.httpClient.execute(this.httpget, this.context);
137                 HttpEntity entity = response.getEntity();
138                 if (entity != null) {
139                     System.out.println(this.httpget.getURI()+": status"+response.getStatusLine().toString());
140                 }
141                 // ensure the connection gets released to the manager
142                 EntityUtils.consume(entity);
143             } catch (Exception ex) {
144                 this.httpget.abort();
145             }finally{
146                 httpget.releaseConnection();
147             }
148         }
149     }
150 }

2、多线程异步请求

  1 package generate.httpclient;
  2 
  3 import java.io.ByteArrayOutputStream;
  4 import java.io.IOException;
  5 import java.io.InputStream;
  6 import java.util.ArrayList;
  7 import java.util.HashMap;
  8 import java.util.List;
  9 import java.util.Map;
 10 import java.util.concurrent.CountDownLatch;
 11 
 12 import org.apache.http.HttpResponse;
 13 import org.apache.http.client.methods.HttpGet;
 14 import org.apache.http.concurrent.FutureCallback;
 15 import org.apache.http.impl.nio.client.DefaultHttpAsyncClient;
 16 import org.apache.http.nio.client.HttpAsyncClient;
 17 import org.apache.http.nio.reactor.IOReactorException;
 18 
 19 public class AsynClient{
 20     /**
 21      * @param args
 22      * @throws IOReactorException
 23      * @throws InterruptedException
 24      */
 25     private List<String> urls;
 26     private HandlerFailThread failHandler;
 27     public AsynClient(List<String> list){
 28         failHandler=new HandlerFailThread();
 29         urls=list;
 30     }
 31     public Map<String,String> asynGet() throws IOReactorException,
 32             InterruptedException {
 33         final HttpAsyncClient httpclient = new DefaultHttpAsyncClient();
 34         httpclient.start();
 35         int urlLength=urls.size();
 36         HttpGet[] requests = new HttpGet[urlLength];
 37         int i=0;
 38         for(String url : urls){
 39             requests[i]=new HttpGet(url);
 40             i++;
 41         }
 42         final CountDownLatch latch = new CountDownLatch(requests.length);
 43         final Map<String, String> responseMap=new HashMap<String, String>();
 44         try {
 45             for (final HttpGet request : requests) {
 46                 httpclient.execute(request, new FutureCallback<HttpResponse>() {
 47 
 48                     public void completed(final HttpResponse response) {
 49                         latch.countDown();
 50                         responseMap.put(request.getURI().toString(), response.getStatusLine().toString());
 51                         try {
 52                             System.out.println(request.getRequestLine() + "->"
 53                                     + response.getStatusLine()+"->");
 54                             //+readInputStream(response.getEntity().getContent())
 55                             
 56                         } catch (IllegalStateException e) {
 57                             failHandler.putFailUrl(request.getURI().toString(),
 58                                     response.getStatusLine().toString());
 59                             e.printStackTrace();
 60                         } catch (Exception e) {
 61                             failHandler.putFailUrl(request.getURI().toString(),
 62                                     response.getStatusLine().toString());
 63                             e.printStackTrace();
 64                         }
 65                     }
 66 
 67                     public void failed(final Exception ex) {
 68                         latch.countDown();
 69                         ex.printStackTrace();
 70                         failHandler.putFailUrl(request.getURI().toString(),
 71                                 ex.getMessage());
 72                     }
 73 
 74                     public void cancelled() {
 75                         latch.countDown();
 76                     }
 77 
 78                 });
 79             }
 80             System.out.println("Doing...");
 81         } finally {
 82             latch.await();
 83             httpclient.shutdown();
 84         }
 85         System.out.println("Done");
 86         failHandler.printFailUrl();
 87         return responseMap;
 88     }
 89     private String readInputStream(InputStream input) throws IOException{
 90         byte[] buffer = new byte[128];
 91         int len = 0;
 92         ByteArrayOutputStream bytes = new ByteArrayOutputStream();
 93         while((len = input.read(buffer)) >= 0) {
 94             bytes.write(buffer, 0, len);
 95         }
 96         return bytes.toString();
 97     }
 98     /**
 99      * Test
100      * @param args
101      */
102     public static void main(String[] args) {
103         List<String> urls=new ArrayList<String>();
104         urls.add("http://127.0.0.1/examples/servlets/");
105         urls.add("http://127.0.0.1/examples/servlets/");
106         urls.add("http://127.0.0.1/examples/servlets/");
107         for(int i=0;i<10;i++){
108             urls.addAll(urls);
109         }
110         System.out.println(urls.size());
111         AsynClient client=new AsynClient(urls);
112         try {
113             client.asynGet();
114         } catch (IOReactorException e) {
115             e.printStackTrace();
116         } catch (InterruptedException e) {
117             e.printStackTrace();
118         }
119         System.out.println("done");
120     }
121 }

创建一个线程记录失败的请求

 1 package generate.httpclient;
 2 
 3 import java.util.HashMap;
 4 import java.util.Map;
 5 
 6 public class HandlerFailThread  extends Thread{
 7     Map<String, String> failUrl=new HashMap<String, String>();
 8     public void putFailUrl(String url,String status){
 9         synchronized (failUrl) {
10             failUrl.put(url,status);
11         }
12     }
13     @Override
14     public void run() {
15         while(true){
16             
17         }
18     }
19     public void printFailUrl(){
20         for(Map.Entry<String, String> m: failUrl.entrySet()){
21             System.out.println("****fail:url:"+m.getKey()+ "  code :"+m.getValue());
22         }
23     }
24 }

  异步请求，也可通过pool管理，例如

 ConnectingIOReactor nio=new DefaultConnectingIOReactor();
  PoolingClientAsyncConnectionManager manager=new PoolingClientAsyncConnectionManager(nio);
  manager.setMaxTotal(1000);
  manager.setDefaultMaxPerRoute(100);
  HttpParams params=new BasicHttpParams();
  /* 连接超时 */
  HttpConnectionParams.setConnectionTimeout(params, 10000);
  /* 请求超时 */
  HttpConnectionParams.setSoTimeout(params, 60*1000);
  DefaultHttpAsyncClient.setDefaultHttpParams(params);
  final HttpAsyncClient httpclient = new DefaultHttpAsyncClient(manager);
  httpclient.start();

HttpClient相关可参看，里面有很多说明与例子

http://hc.apache.org/httpcomponents-client-ga/tutorial/html/connmgmt.html