官网
Apache HttpComponents – Apache HttpComponents
依赖
<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents.client5</groupId>
<artifactId>httpclient5</artifactId>
<version>5.1.3</version>
</dependency>
</dependencies>
源码
HttpClient类图
InternalHttpClient.png
execute流程
InternalHttpClient
protected CloseableHttpResponse doExecute(
final HttpHost target,
final ClassicHttpRequest request,
final HttpContext context) throws IOException {
Args.notNull(request, "HTTP request");
try {
// 上下文
final HttpClientContext localcontext = HttpClientContext.adapt(
context != null context : new BasicHttpContext());
// 请求超时、连接超时、响应超时等
RequestConfig config = null;
if (request instanceof Configurable) {
config = ((Configurable) request).getConfig();
}
if (config != null) {
localcontext.setRequestConfig(config);
}
setupContext(localcontext);
final HttpRoute route = determineRoute(
target != null target : RoutingSupport.determineHost(request),
localcontext);
final String exchangeId = ExecSupport.getNextExchangeId();
localcontext.setExchangeId(exchangeId);
if (LOG.isDebugEnabled()) {
LOG.debug("{} preparing request execution", exchangeId);
}
final ExecRuntime execRuntime = new InternalExecRuntime(LOG, connManager, requestExecutor,
request instanceof CancellableDependency (CancellableDependency) request : null);
final ExecChain.Scope scope = new ExecChain.Scope(exchangeId, route, request, execRuntime, localcontext);
// 几个ExecChainElement连接成处理链
// 实际处理是ExecChainHandler
final ClassicHttpResponse response = this.execChain.execute(ClassicRequestBuilder.copy(request).build(), scope);
return CloseableHttpResponse.adapt(response);
} catch (final HttpException httpException) {
throw new ClientProtocolException(httpException.getMessage(), httpException);
}
}
HttpRequestRetryExec
public class HttpRequestRetryExec implements ExecChainHandler {
private static final Logger LOG = LoggerFactory.getLogger(HttpRequestRetryExec.class);
private final HttpRequestRetryStrategy retryStrategy;
public HttpRequestRetryExec(
final HttpRequestRetryStrategy retryStrategy) {
Args.notNull(retryStrategy, "retryStrategy");
this.retryStrategy = retryStrategy;
}
@Override
public ClassicHttpResponse execute(
final ClassicHttpRequest request,
final Scope scope,
final ExecChain chain) throws IOException, HttpException {
Args.notNull(request, "request");
Args.notNull(scope, "scope");
final String exchangeId = scope.exchangeId;
final HttpRoute route = scope.route;
final HttpClientContext context = scope.clientContext;
ClassicHttpRequest currentRequest = request;
// 这个版本从1开始计数
for (int execCount = 1;; execCount++) {
final ClassicHttpResponse response;
try {
response = chain.proceed(currentRequest, scope);
} catch (final IOException ex) {
if (scope.execRuntime.isExecutionAborted()) {
throw new RequestFailedException("Request aborted");
}
// 返回的是null
final HttpEntity requestEntity = request.getEntity();
if (requestEntity != null && !requestEntity.isRepeatable()) {
if (LOG.isDebugEnabled()) {
LOG.debug("{} cannot retry non-repeatable request", exchangeId);
}
throw ex;
}
// IO异常时的retry判断逻辑
if (retryStrategy.retryRequest(request, ex, execCount, context)) {
if (LOG.isDebugEnabled()) {
LOG.debug("{} {}", exchangeId, ex.getMessage(), ex);
}
if (LOG.isInfoEnabled()) {
LOG.info("Recoverable I/O exception ({}) caught when processing request to {}",
ex.getClass().getName(), route);
}
currentRequest = ClassicRequestBuilder.copy(scope.originalRequest).build();
continue;
} else {
if (ex instanceof NoHttpResponseException) {
final NoHttpResponseException updatedex = new NoHttpResponseException(
route.getTargetHost().toHostString() + " failed to respond");
updatedex.setStackTrace(ex.getStackTrace());
throw updatedex;
}
throw ex;
}
}
try {
final HttpEntity entity = request.getEntity();
if (entity != null && !entity.isRepeatable()) {
if (LOG.isDebugEnabled()) {
LOG.debug("{} cannot retry non-repeatable request", exchangeId);
}
return response;
}
// 从服务端获得响应时判断是否retry，如响应429限频
// 4.5版本只会进行IO异常重试
if (retryStrategy.retryRequest(response, execCount, context)) {
// 重试间隔时间需要小于响应时间
final TimeValue nextInterval = retryStrategy.getRetryInterval(response, execCount, context);
// Make sure the retry interval does not exceed the response timeout
if (TimeValue.isPositive(nextInterval)) {
final RequestConfig requestConfig = context.getRequestConfig();
final Timeout responseTimeout = requestConfig.getResponseTimeout();
if (responseTimeout != null && nextInterval.compareTo(responseTimeout) > 0) {
return response;
}
}
response.close();
if (TimeValue.isPositive(nextInterval)) {
try {
if (LOG.isDebugEnabled()) {
LOG.debug("{} wait for {}", exchangeId, nextInterval);
}
nextInterval.sleep();
} catch (final InterruptedException e) {
Thread.currentThread().interrupt();
throw new InterruptedIOException();
}
}
currentRequest = ClassicRequestBuilder.copy(scope.originalRequest).build();
} else {
return response;
}
} catch (final RuntimeException ex) {
response.close();
throw ex;
}
}
}
}
HttpRequestRetryStrategy
public interface HttpRequestRetryStrategy {
/**
* Determines if a method should be retried after an I/O exception
* occurred during execution.
*
* @param request the request failed due to an I/O exception
* @param exception the exception that occurred
* @param execCount the number of times this method has been
* unsuccessfully executed
* @param context the context for the request execution
*
* @return {@code true} if the request should be retried, {@code false}
* otherwise
*/
boolean retryRequest(HttpRequest request, IOException exception, int execCount, HttpContext context);
/**
* Determines if a method should be retried given the response from
* the target server.
*
* @param response the response from the target server
* @param execCount the number of times this method has been
* unsuccessfully executed
* @param context the context for the request execution
*
* @return {@code true} if the request should be retried, {@code false}
* otherwise
*/
boolean retryRequest(HttpResponse response, int execCount, HttpContext context);
/**
* Determines the retry interval between subsequent retries.
*
* @param response the response from the target server
* @param execCount the number of times this method has been
* unsuccessfully executed
* @param context the context for the request execution
*
* @return the retry interval between subsequent retries
*/
TimeValue getRetryInterval(HttpResponse response, int execCount, HttpContext context);
默认的实现中 interval 需要小于响应超时时间
请求时可能发生IO异常，如连接超时或者响应超时
如果从服务端正确获得响应，不同的响应码代表不同场景
上述两种大的场景都需要考虑重试机制，
①如连接超时设置的比较长，又重试多次可能影响任务执行的时间引起其他问题
②响应码429，得考虑sleep一定时间再重试
③401等即使重试大概率没用
案例
package com.itachi.http;
import org.apache.hc.client5.http.classic.methods.HttpGet;
import org.apache.hc.client5.http.config.RequestConfig;
import org.apache.hc.client5.http.impl.DefaultConnectionKeepAliveStrategy;
import org.apache.hc.client5.http.impl.DefaultHttpRequestRetryStrategy;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse;
import org.apache.hc.client5.http.impl.classic.HttpClientBuilder;
import org.apache.hc.client5.http.impl.classic.HttpClients;
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager;
import org.apache.hc.client5.http.socket.ConnectionSocketFactory;
import org.apache.hc.client5.http.socket.PlainConnectionSocketFactory;
import org.apache.hc.client5.http.ssl.SSLConnectionSocketFactory;
import org.apache.hc.core5.http.URIScheme;
import org.apache.hc.core5.http.config.Registry;
import org.apache.hc.core5.http.config.RegistryBuilder;
import org.apache.hc.core5.util.TimeValue;
import org.apache.hc.core5.util.Timeout;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;
import java.io.IOException;
import java.security.KeyManagementException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
public class Test {
@org.junit.Test
public void testHttp() {
HttpClientBuilder custom = HttpClients.custom();
// 连接池设置
// SocketFactory + 最大连接数 + 每个rout的最大连接数
// 连接池中的连接默认一直存活
SSLConnectionSocketFactory sslConnectionSocketFactory = new SSLConnectionSocketFactory(testSSL());
Registry<ConnectionSocketFactory> ssl = RegistryBuilder.<ConnectionSocketFactory>create()
.register(URIScheme.HTTP.id, PlainConnectionSocketFactory.getSocketFactory())
.register(URIScheme.HTTPS.id, sslConnectionSocketFactory)
.build();
PoolingHttpClientConnectionManager poolingHttpClientConnectionManager = new PoolingHttpClientConnectionManager(ssl);
custom.setConnectionManager(poolingHttpClientConnectionManager);
poolingHttpClientConnectionManager.setMaxTotal(200);
poolingHttpClientConnectionManager.setDefaultMaxPerRoute(50);
// IO异常和收到请求后重试逻辑
DefaultHttpRequestRetryStrategy defaultHttpRequestRetryStrategy = new DefaultHttpRequestRetryStrategy(1, TimeValue.ofSeconds(1));
custom.setRetryStrategy(defaultHttpRequestRetryStrategy);
//
custom.setKeepAliveStrategy(DefaultConnectionKeepAliveStrategy.INSTANCE);
// 请求参数：连接超时、响应超时
RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(Timeout.ofSeconds(30))
.setConnectionKeepAlive(TimeValue.ofSeconds(30))
.setResponseTimeout(Timeout.ofSeconds(10)).build();
custom.setDefaultRequestConfig(requestConfig);
CloseableHttpClient client = custom.build();
HttpGet httpGet = new HttpGet("https://www.baidu.com");
try {
CloseableHttpResponse execute = client.execute(httpGet);
System.out.println(execute.getCode());
} catch (IOException e) {
e.printStackTrace();
}
}
public SSLContext testSSL() {
try {
SSLContext tls = SSLContext.getInstance("TLS");
TrustManager[] trust = {new X509TrustManager() {
public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException {
}
public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException {
}
public X509Certificate[] getAcceptedIssuers() {
return new X509Certificate[0];
}
}};
tls.init(null, trust, null);
return tls;
} catch (NoSuchAlgorithmException e) {
e.printStackTrace();
} catch (KeyManagementException e) {
e.printStackTrace();
}
return null;
}
}