前言
现在大多数Java项目开发中,经常都用通过HTTP协议来调用网络资源数据(1、爬虫爬取网页数据;2、请求第三方系统进行数据交互等),虽然JDK8及以前的版本,也提供了响应的请求工具包,但是使用起来很不灵活,所以大多数都是采用Apache的HttpClient包来封装自己的请求工具类,方便整个项目开发使用。
使用流程
1、引入maven依赖
<dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpclient</artifactId> <version>4.5.13</version> </dependency> <dependency> <groupId>com.alibaba</groupId> <artifactId>fastjson</artifactId> <version>1.2.75</version> </dependency>
2、HttpClient使用
2.1 创建请求客户端对象
/* 1、创建自定义请求客户端构建对象 */ HttpClientBuilder httpClientBuilder = HttpClients.custom(); // 创建连接池,并对连接池进行设置后赋值给请求对象构造器 HttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(); httpClientBuilder.setConnectionManager(cm); CloseableHttpClient build = httpClientBuilder.build(); /* 2、创建默认的请求客户端构建对象 */ CloseableHttpClient request2 = HttpClients.createDefault();
2.2 创建连接池管理对象
// 创建请求连接池管理,可创建空的,也可以创建带注册对象(注册对象示例:https的连接认证注册) PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(); // 设置连接池最大连接数(设置时可以设置部署机器的CPU线程数) cm.setMaxTotal(8); // 设置每个路由最大默认连接数(可以认为一个域名就是路由) cm.setDefaultMaxPerRoute(8);
2.3 请求方式对象
// get请求对象 HttpGet httpGet = new HttpGet(url); // post请求对象 HttpPost httpPost = new HttpPost(url);
2.4 请求头设置
// 设置用户代理为浏览器 httpGet.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.71 Safari/537.36 Edg/94.0.992.38"); // 设置请求的数据类型 httpPost.setHeader("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8"); httpPost.setHeader("Content-Type", "application/json;charset=UTF-8");
2.5 参数设置
注意:随着请求方式的不同,传参方式也大不相同
/* 1、get请求 */ String url = "url"; Map<String, String> params = new HashMap<>(); params.put("username", "chris"); URIBuilder uriBuilder = new URIBuilder(url); if (params!=null) { // 判空操作要有,避免空指针异常 for (String paramKey : params.keySet()) { uriBuilder.addParameter(paramKey, params.get(paramKey)); } } HttpGet httpGet = new HttpGet(uriBuilder.build());
/* 2、post请求(表单形式:application/x-www-form-urlencoded) */ String url = "url"; Map<String, String> params = new HashMap<>(); params.put("username", "chris"); HttpPost httpPost = new HttpPost(url); if (params!=null) { // 判空操作要有,避免空指针异常 List<NameValuePair> nvList = new ArrayList<>(params.size()); NameValuePair nv = null; for (String paramKey : params.keySet()) { // 构建参数键值对象 nv = new BasicNameValuePair(paramKey, params.get(paramKey)); nvList.add(nv); } // 传送参数的对象 HttpEntity paramsEntity = new UrlEncodedFormEntity(nvList, StandardCharsets.UTF_8); // 设置参数 httpPost.setEntity(paramsEntity); }
/* 3、post请求(json形式:application/json) */ String url = "url"; Map<String, String> params = new HashMap<>(); params.put("username", "chris"); HttpPost httpPost = new HttpPost(url); if (params!=null) { // 判空操作要有,避免空指针异常 // 需要将请求对象转换为json字符串形式 HttpEntity paramEntity = new StringEntity(JSON.toJSONString(params), StandardCharsets.UTF_8); httpPost.setEntity(paramEntity); }
2.6 执行请求并处理返回对象
CloseableHttpResponse response = null; try { // 执行请求 response = closeableHttpClient.execute(httpPost); int statusCode = response.getStatusLine().getStatusCode(); // 判断请求响应是否成功 if (statusCode== HttpStatus.SC_OK) { // 处理响应数据并返回 HttpEntity entity = response.getEntity(); return EntityUtils.toString(entity, StandardCharsets.UTF_8); } else { log.error("请求地址({})失败:{}", url, statusCode); } } catch (IOException e) { log.error("请求地址({})失败", url, e); throw new RuntimeException("请求地址("+url+")失败"); } finally { // 确认数据消费并关闭http响应对象 HttpClientUtils.closeQuietly(response); }
3、书写请求工具类
/** * http请求工具类 */ @Slf4j public class HttpUtil { /** * 请求连接构造对象 */ private static final HttpClientBuilder httpClientBuilder = HttpClients.custom(); /** * 连接池最大连接数 */ private static final int MAX_TOTAL = 8; /** * 每个路由最大默认连接数 */ private static final int DEFAULT_MAX_RER_ROUTE = 8; /** * 获取连接获取超时时间 */ private static final int CONNECTION_REQUEST_TIMEOUT = 2000; /** * 连接超时时间 */ private static final int CONNECTION_TIMEOUT = 2000; /** * 数据响应超时时间 */ private static final int SOCKET_TIMEOUT = 10000; static { /* 1、绕开不安全的https请求的证书验证(不需要可以注释,然后使用空参数的PoolingHttpClientConnectionManager构造连接池管理对象) */ Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create() .register("http", PlainConnectionSocketFactory.INSTANCE) .register("https", trustHttpsCertificates()) .build(); /* 2、创建请求连接池管理 */ PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(registry); // 设置连接池最大连接数 cm.setMaxTotal(MAX_TOTAL); // 设置每个路由最大默认连接数 cm.setDefaultMaxPerRoute(DEFAULT_MAX_RER_ROUTE); httpClientBuilder.setConnectionManager(cm); /* 3、设置默认请求配置 */ RequestConfig requestConfig = RequestConfig.custom() .setConnectionRequestTimeout(CONNECTION_REQUEST_TIMEOUT) // 设置获取连接获取超时时间 .setConnectTimeout(CONNECTION_TIMEOUT) // 设置连接超时时间 .setSocketTimeout(SOCKET_TIMEOUT) // 设置数据响应超时时间 .build(); httpClientBuilder.setDefaultRequestConfig(requestConfig); } /** * 执行get请求(网页) * @param url 请求地址(含有特殊符号需要URLEncoder编码) * @param headers 请求头参数 * @return 响应数据 */ public static String getPage(String url, Map<String, String> headers) { CloseableHttpClient closeableHttpClient = httpClientBuilder.build(); HttpGet httpGet = new HttpGet(url); // 请求头设置,如果常用的请求头设置,也可以写死,特殊的请求才传入 httpGet.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.71 Safari/537.36 Edg/94.0.992.38"); if (headers != null) { for (String headerKey : headers.keySet()) { httpGet.setHeader(headerKey, headers.get(headerKey)); } } CloseableHttpResponse response = null; try { response = closeableHttpClient.execute(httpGet); int statusCode = response.getStatusLine().getStatusCode(); if (statusCode== HttpStatus.SC_OK) { // 请求响应成功 HttpEntity entity = response.getEntity(); return EntityUtils.toString(entity, StandardCharsets.UTF_8); } else { log.error("请求地址({})失败:{}", url, statusCode); } } catch (Exception e) { log.error("请求地址({})失败", url, e); throw new RuntimeException("请求地址("+url+")失败"); } finally { HttpClientUtils.closeQuietly(response); } return null; } /** * 执行post请求(form表单) * @param url 请求地址 * @param headers 请求头参数 * @return 响应数据 */ public static String postForm(String url, Map<String, String> headers, Map<String, String> params) { CloseableHttpClient closeableHttpClient = httpClientBuilder.build(); HttpPost httpPost = new HttpPost(url); // 请求头设置,如果常用的请求头设置,也可以写死,特殊的请求才传入 httpPost.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.71 Safari/537.36 Edg/94.0.992.38"); httpPost.setHeader("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8"); if (headers != null) { for (String headerKey : headers.keySet()) { httpPost.setHeader(headerKey, headers.get(headerKey)); } } // 设置请求参数 if (params!=null) { List<NameValuePair> nvList = new ArrayList<>(params.size()); for (String paramKey : params.keySet()) { NameValuePair nv = new BasicNameValuePair(paramKey, params.get(paramKey)); nvList.add(nv); } HttpEntity paramsEntity = new UrlEncodedFormEntity(nvList, StandardCharsets.UTF_8); httpPost.setEntity(paramsEntity); } CloseableHttpResponse response = null; try { response = closeableHttpClient.execute(httpPost); int statusCode = response.getStatusLine().getStatusCode(); if (statusCode== HttpStatus.SC_OK) { // 请求响应成功 HttpEntity entity = response.getEntity(); return EntityUtils.toString(entity, StandardCharsets.UTF_8); } else { log.error("请求地址({})失败:{}", url, statusCode); } } catch (IOException e) { log.error("请求地址({})失败", url, e); throw new RuntimeException("请求地址("+url+")失败"); } finally { HttpClientUtils.closeQuietly(response); } return null; } /** * 执行post请求(接口) * @param url 请求地址 * @param headers 请求头参数 * @return 响应数据 */ public static String getJson(String url, Map<String, String> headers) { CloseableHttpClient closeableHttpClient = httpClientBuilder.build(); HttpGet httpGet = new HttpGet(url); // 请求头设置,如果常用的请求头设置,也可以写死,特殊的请求才传入 if (headers != null) { for (String headerKey : headers.keySet()) { httpGet.setHeader(headerKey, headers.get(headerKey)); } } CloseableHttpResponse response = null; try { response = closeableHttpClient.execute(httpGet); int statusCode = response.getStatusLine().getStatusCode(); if (statusCode== HttpStatus.SC_OK) { // 请求响应成功 HttpEntity entity = response.getEntity(); return EntityUtils.toString(entity, StandardCharsets.UTF_8); } else { log.error("请求地址({})失败:{}", url, statusCode); } } catch (IOException e) { log.error("请求地址({})失败", url, e); throw new RuntimeException("请求地址("+url+")失败"); } finally { HttpClientUtils.closeQuietly(response); } return null; } /** * 执行post请求(接口) * @param url 请求地址 * @param headers 请求头参数 * @return 响应数据 */ public static String postJson(String url, Map<String, String> headers, Map<String, String> params) { CloseableHttpClient closeableHttpClient = httpClientBuilder.build(); HttpPost httpPost = new HttpPost(url); // 请求头设置,如果常用的请求头设置,也可以写死,特殊的请求才传入 httpPost.setHeader("Content-Type", "application/json;charset=UTF-8"); if (headers != null) { for (String headerKey : headers.keySet()) { httpPost.setHeader(headerKey, headers.get(headerKey)); } } if (params!=null) { HttpEntity paramEntity = new StringEntity(JSON.toJSONString(params), StandardCharsets.UTF_8); httpPost.setEntity(paramEntity); } CloseableHttpResponse response = null; try { response = closeableHttpClient.execute(httpPost); int statusCode = response.getStatusLine().getStatusCode(); if (statusCode== HttpStatus.SC_OK) { // 请求响应成功 HttpEntity entity = response.getEntity(); return EntityUtils.toString(entity, StandardCharsets.UTF_8); } else { log.error("请求地址({})失败:{}", url, statusCode); } } catch (IOException e) { log.error("请求地址({})失败", url, e); throw new RuntimeException("请求地址("+url+")失败"); } finally { HttpClientUtils.closeQuietly(response); } return null; } /** * 构建https安全连接工厂 * @return 安全连接工厂 */ private static ConnectionSocketFactory trustHttpsCertificates() { SSLContextBuilder sslContextBuilder = new SSLContextBuilder(); try { sslContextBuilder.loadTrustMaterial(null, new TrustStrategy() { @Override public boolean isTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException { return true; } }); SSLContext sslContext = sslContextBuilder.build(); SSLConnectionSocketFactory sslConnectionSocketFactory = new SSLConnectionSocketFactory(sslContext, new String[]{"SSLv2Hello", "SSLv3", "TLSv1", "TLSv1.1", "TLSv1.2"}, // 支持的https安全认证协议 null, NoopHostnameVerifier.INSTANCE); return sslConnectionSocketFactory; } catch (Exception e) { log.error("构建安全连接工厂失败", e); throw new RuntimeException("构建安全连接工厂失败"); } } }
4、额外说明
- 项目中HttpClient基本最好构建请求连接池进行请求,提升性能,加快请求速度;
- 如果你的应用不请求不信任的https连接,则不需要绕过https安全认证(不需要使用到trustHttpsCertificates方法);
- 项目请求在工具类一般可以确定请求头,可以写死,从而少传入一个参数;(特殊要求传入的请求才保留该参数设置)
- 每次请求后一定要确认消费和关闭响应;(HttpClientUtils.closeQuietly(response))
- 部分爬虫需要设置代理,使用示例如下:
HttpGet httpGet = new HttpGet(url); HttpHost proxy = new HttpHost("49.70.17.48", 8888); RequestConfig config = RequestConfig.custom() .setProxy(proxy) //设置代理 .setConnectTimeout(2000) // 设置HTTP连接超时时间 .setSocketTimeout(3000) // 设置数据响应超时时间 .setConnectionRequestTimeout(2000) // 设置从连接池获取连接的超时时间 .build(); httpGet.setConfig(config);
成功 = 正确的选择 + 实际的行动 + 长期的坚持;