前言:
httpclient(4.5.x)默认是启动连接池的, 其降低时耗(避免连接初3次握手, 以及关闭4次握手的消耗), 显著提升高并发处理能力(大量减少time_wait), 确实扮演了重要的角色. 但是封装httpclient, 需要了解不少细节, 还要根据业务合理配置参数.
这里结合这段时间深入httpclient(4.5.x)源码分析, 结合网上的代码案例, 以及线下测试的结果. 尝试写一个可用的的httpclient封装类, 兼顾性能, 接口友好. 感谢cctv, ^_^.
相关文章:
1. HttpClient官方sample代码的深入分析(连接池)
第一版本:
啥也不说了, 直接贴代码了.
public class PooledHttpClientAdaptor {
private static final int DEFAULT_POOL_MAX_TOTAL = 200;
private static final int DEFAULT_POOL_MAX_PER_ROUTE = 200;
private static final int DEFAULT_CONNECT_TIMEOUT = 500;
private static final int DEFAULT_CONNECT_REQUEST_TIMEOUT = 500;
private static final int DEFAULT_SOCKET_TIMEOUT = 2000;
private PoolingHttpClientConnectionManager gcm = null;
private CloseableHttpClient httpClient = null;
// 连接池的最大连接数
private final int maxTotal;
// 连接池按route配置的最大连接数
private final int maxPerRoute;
// tcp connect的超时时间
private final int connectTimeout;
// 从连接池获取连接的超时时间
private final int connectRequestTimeout;
// tcp io的读写超时时间
private final int socketTimeout;
public PooledHttpClientAdaptor() {
this(
PooledHttpClientAdaptor.DEFAULT_POOL_MAX_TOTAL,
PooledHttpClientAdaptor.DEFAULT_POOL_MAX_PER_ROUTE,
PooledHttpClientAdaptor.DEFAULT_CONNECT_TIMEOUT,
PooledHttpClientAdaptor.DEFAULT_CONNECT_REQUEST_TIMEOUT,
PooledHttpClientAdaptor.DEFAULT_SOCKET_TIMEOUT
);
}
public PooledHttpClientAdaptor(
int maxTotal,
int maxPerRoute,
int connectTimeout,
int connectRequestTimeout,
int socketTimeout
) {
this.maxTotal = maxTotal;
this.maxPerRoute = maxPerRoute;
this.connectTimeout = connectTimeout;
this.connectRequestTimeout = connectRequestTimeout;
this.socketTimeout = socketTimeout;
Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create()
.register("http", PlainConnectionSocketFactory.getSocketFactory())
.register("https", SSLConnectionSocketFactory.getSocketFactory())
.build();
gcm = new PoolingHttpClientConnectionManager(registry);
gcm.setMaxTotal(this.maxTotal);
gcm.setDefaultMaxPerRoute(this.maxPerRoute);
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(this.connectTimeout) // 设置连接超时
.setSocketTimeout(this.socketTimeout) // 设置读取超时
.setConnectionRequestTimeout(this.connectRequestTimeout) // 设置从连接池获取连接实例的超时
.build();
HttpClientBuilder httpClientBuilder = HttpClients.custom();
httpClient = httpClientBuilder
.setConnectionManager(gcm)
.setDefaultRequestConfig(requestConfig)
.build();
}
public String doGet(String url) {
return this.doGet(url, Collections.EMPTY_MAP, Collections.EMPTY_MAP);
}
public String doGet(String url, Map<String, Object> params) {
return this.doGet(url, Collections.EMPTY_MAP, params);
}
public String doGet(String url,
Map<String, String> headers,
Map<String, Object> params
) {
// *) 构建GET请求头
String apiUrl = getUrlWithParams(url, params);
HttpGet httpGet = new HttpGet(apiUrl);
// *) 设置header信息
if ( headers != null && headers.size() > 0 ) {
for (Map.Entry<String, String> entry : headers.entrySet()) {
httpGet.addHeader(entry.getKey(), entry.getValue());
}
}
CloseableHttpResponse response = null;
try {
response = httpClient.execute(httpGet);
if (response == null || response.getStatusLine() == null) {
return null;
}
int statusCode = response.getStatusLine().getStatusCode();
if ( statusCode == HttpStatus.SC_OK ) {
HttpEntity entityRes = response.getEntity();
if (entityRes != null) {
return EntityUtils.toString(entityRes, "UTF-8");
}
}
return null;
} catch (IOException e) {
} finally {
if ( response != null ) {
try {
response.close();
} catch (IOException e) {
}
}
}
return null;
}
public String doPost(String apiUrl, Map<String, Object> params) {
return this.doPost(apiUrl, Collections.EMPTY_MAP, params);
}
public String doPost(String apiUrl,
Map<String, String> headers,
Map<String, Object> params
) {
HttpPost httpPost = new HttpPost(apiUrl);
// *) 配置请求headers
if ( headers != null && headers.size() > 0 ) {
for (Map.Entry<String, String> entry : headers.entrySet()) {
httpPost.addHeader(entry.getKey(), entry.getValue());
}
}
// *) 配置请求参数
if ( params != null && params.size() > 0 ) {
HttpEntity entityReq = getUrlEncodedFormEntity(params);
httpPost.setEntity(entityReq);
}
CloseableHttpResponse response = null;
try {
response = httpClient.execute(httpPost);
if (response == null || response.getStatusLine() == null) {
return null;
}
int statusCode = response.getStatusLine().getStatusCode();
if ( statusCode == HttpStatus.SC_OK ) {
HttpEntity entityRes = response.getEntity();
if ( entityRes != null ) {
return EntityUtils.toString(entityRes, "UTF-8");
}
}
return null;
} catch (IOException e) {
} finally {
if (response != null) {
try {
response.close();
} catch (IOException e) {
}
}
}
return null;
}
private HttpEntity getUrlEncodedFormEntity(Map<String, Object> params) {
List<NameValuePair> pairList = new ArrayList<NameValuePair>(params.size());
for (Map.Entry<String, Object> entry : params.entrySet()) {
NameValuePair pair = new BasicNameValuePair(entry.getKey(), entry
.getValue().toString());
pairList.add(pair);
}
return new UrlEncodedFormEntity(pairList, Charset.forName("UTF-8"));
}
private String getUrlWithParams(String url, Map<String, Object> params) {
boolean first = true;
StringBuilder sb = new StringBuilder(url);
for (String key : params.keySet()) {
char ch = '&';
if (first == true) {
ch = '?';
first = false;
}
String value = params.get(key).toString();
try {
String sval = URLEncoder.encode(value, "UTF-8");
sb.append(ch).append(key).append("=").append(sval);
} catch (UnsupportedEncodingException e) {
}
}
return sb.toString();
}
}
存在问题&解决方案
这个版本基本没啥问题, 但是当流量为0时, 你会发现存在处于ClOSE_WAIT的连接. 究其原因是, httpclient清理过期/被动关闭的socket, 是采用懒惰清理的策略. 它是在连接从连接池取出使用的时候, 检测状态并做相应处理. 如果没有流量, 那这些socket将一直处于CLOSE_WAIT(半连接的状态), 系统资源被浪费.
不过解决方案也相当的简单, 官方的建议引入一个清理线程, 定期主动处理过期/空闲连接, 这样就OK了.
private class IdleConnectionMonitorThread extends Thread {
private final HttpClientConnectionManager connMgr;
private volatile boolean exitFlag = false;
public IdleConnectionMonitorThread(HttpClientConnectionManager connMgr) {
this.connMgr = connMgr;
setDaemon(true);
}
@Override
public void run() {
while (!this.exitFlag) {
synchronized (this) {
try {
this.wait(2000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
// 关闭失效的连接
connMgr.closeExpiredConnections();
// 可选的, 关闭30秒内不活动的连接
connMgr.closeIdleConnections(30, TimeUnit.SECONDS);
}
}
public void shutdown() {
this.exitFlag = true;
synchronized (this) {
notify();
}
}
}
最终版本
直接整合贴代码, ^_^.
public class PooledHttpClientAdaptor {
private static final int DEFAULT_POOL_MAX_TOTAL = 200;
private static final int DEFAULT_POOL_MAX_PER_ROUTE = 200;
private static final int DEFAULT_CONNECT_TIMEOUT = 500;
private static final int DEFAULT_CONNECT_REQUEST_TIMEOUT = 500;
private static final int DEFAULT_SOCKET_TIMEOUT = 2000;
private PoolingHttpClientConnectionManager gcm = null;
private CloseableHttpClient httpClient = null;
private IdleConnectionMonitorThread idleThread = null;
// 连接池的最大连接数
private final int maxTotal;
// 连接池按route配置的最大连接数
private final int maxPerRoute;
// tcp connect的超时时间
private final int connectTimeout;
// 从连接池获取连接的超时时间
private final int connectRequestTimeout;
// tcp io的读写超时时间
private final int socketTimeout;
public PooledHttpClientAdaptor() {
this(
PooledHttpClientAdaptor.DEFAULT_POOL_MAX_TOTAL,
PooledHttpClientAdaptor.DEFAULT_POOL_MAX_PER_ROUTE,
PooledHttpClientAdaptor.DEFAULT_CONNECT_TIMEOUT,
PooledHttpClientAdaptor.DEFAULT_CONNECT_REQUEST_TIMEOUT,
PooledHttpClientAdaptor.DEFAULT_SOCKET_TIMEOUT
);
}
public PooledHttpClientAdaptor(
int maxTotal,
int maxPerRoute,
int connectTimeout,
int connectRequestTimeout,
int socketTimeout
) {
this.maxTotal = maxTotal;
this.maxPerRoute = maxPerRoute;
this.connectTimeout = connectTimeout;
this.connectRequestTimeout = connectRequestTimeout;
this.socketTimeout = socketTimeout;
Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create()
.register("http", PlainConnectionSocketFactory.getSocketFactory())
.register("https", SSLConnectionSocketFactory.getSocketFactory())
.build();
this.gcm = new PoolingHttpClientConnectionManager(registry);
this.gcm.setMaxTotal(this.maxTotal);
this.gcm.setDefaultMaxPerRoute(this.maxPerRoute);
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(this.connectTimeout) // 设置连接超时
.setSocketTimeout(this.socketTimeout) // 设置读取超时
.setConnectionRequestTimeout(this.connectRequestTimeout) // 设置从连接池获取连接实例的超时
.build();
HttpClientBuilder httpClientBuilder = HttpClients.custom();
httpClient = httpClientBuilder
.setConnectionManager(this.gcm)
.setDefaultRequestConfig(requestConfig)
.build();
idleThread = new IdleConnectionMonitorThread(this.gcm);
idleThread.start();
}
public String doGet(String url) {
return this.doGet(url, Collections.EMPTY_MAP, Collections.EMPTY_MAP);
}
public String doGet(String url, Map<String, Object> params) {
return this.doGet(url, Collections.EMPTY_MAP, params);
}
public String doGet(String url,
Map<String, String> headers,
Map<String, Object> params
) {
// *) 构建GET请求头
String apiUrl = getUrlWithParams(url, params);
HttpGet httpGet = new HttpGet(apiUrl);
// *) 设置header信息
if ( headers != null && headers.size() > 0 ) {
for (Map.Entry<String, String> entry : headers.entrySet()) {
httpGet.addHeader(entry.getKey(), entry.getValue());
}
}
CloseableHttpResponse response = null;
try {
response = httpClient.execute(httpGet);
if (response == null || response.getStatusLine() == null) {
return null;
}
int statusCode = response.getStatusLine().getStatusCode();
if ( statusCode == HttpStatus.SC_OK ) {
HttpEntity entityRes = response.getEntity();
if (entityRes != null) {
return EntityUtils.toString(entityRes, "UTF-8");
}
}
return null;
} catch (IOException e) {
} finally {
if ( response != null ) {
try {
response.close();
} catch (IOException e) {
}
}
}
return null;
}
public String doPost(String apiUrl, Map<String, Object> params) {
return this.doPost(apiUrl, Collections.EMPTY_MAP, params);
}
public String doPost(String apiUrl,
Map<String, String> headers,
Map<String, Object> params
) {
HttpPost httpPost = new HttpPost(apiUrl);
// *) 配置请求headers
if ( headers != null && headers.size() > 0 ) {
for (Map.Entry<String, String> entry : headers.entrySet()) {
httpPost.addHeader(entry.getKey(), entry.getValue());
}
}
// *) 配置请求参数
if ( params != null && params.size() > 0 ) {
HttpEntity entityReq = getUrlEncodedFormEntity(params);
httpPost.setEntity(entityReq);
}
CloseableHttpResponse response = null;
try {
response = httpClient.execute(httpPost);
if (response == null || response.getStatusLine() == null) {
return null;
}
int statusCode = response.getStatusLine().getStatusCode();
if ( statusCode == HttpStatus.SC_OK ) {
HttpEntity entityRes = response.getEntity();
if ( entityRes != null ) {
return EntityUtils.toString(entityRes, "UTF-8");
}
}
return null;
} catch (IOException e) {
} finally {
if (response != null) {
try {
response.close();
} catch (IOException e) {
}
}
}
return null;
}
private HttpEntity getUrlEncodedFormEntity(Map<String, Object> params) {
List<NameValuePair> pairList = new ArrayList<NameValuePair>(params.size());
for (Map.Entry<String, Object> entry : params.entrySet()) {
NameValuePair pair = new BasicNameValuePair(entry.getKey(), entry
.getValue().toString());
pairList.add(pair);
}
return new UrlEncodedFormEntity(pairList, Charset.forName("UTF-8"));
}
private String getUrlWithParams(String url, Map<String, Object> params) {
boolean first = true;
StringBuilder sb = new StringBuilder(url);
for (String key : params.keySet()) {
char ch = '&';
if (first == true) {
ch = '?';
first = false;
}
String value = params.get(key).toString();
try {
String sval = URLEncoder.encode(value, "UTF-8");
sb.append(ch).append(key).append("=").append(sval);
} catch (UnsupportedEncodingException e) {
}
}
return sb.toString();
}
public void shutdown() {
idleThread.shutdown();
}
// 监控有异常的链接
private class IdleConnectionMonitorThread extends Thread {
private final HttpClientConnectionManager connMgr;
private volatile boolean exitFlag = false;
public IdleConnectionMonitorThread(HttpClientConnectionManager connMgr) {
this.connMgr = connMgr;
setDaemon(true);
}
@Override
public void run() {
while (!this.exitFlag) {
synchronized (this) {
try {
this.wait(2000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
// 关闭失效的连接
connMgr.closeExpiredConnections();
// 可选的, 关闭30秒内不活动的连接
connMgr.closeIdleConnections(30, TimeUnit.SECONDS);
}
}
public void shutdown() {
this.exitFlag = true;
synchronized (this) {
notify();
}
}
}
}
总结:
其实没啥难道, 主要就是买个安心, 这样写是安全的, 是经得起线上考验的.