How to Test Apache HttpClient in the Context of AEM

If you’ve ever written a proxy servlet in AEM, chances are you’ve used Apache’s HttpComponents library. While a great library, there are not many resources online for how to test it when used inside your code. If you have not seen my post, The Ultimate Code Quality Setup for your AEM project , you should check it out. The test code in this post is written with jUnit5, although most of the concepts here apply to jUnit4 as well. Now onto the problem:

Let’s look at an example servlet that proxies to a hard-coded url.
// VERY dudamentary code to illustrate the point
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import javax.servlet.Servlet;
import javax.servlet.ServletException;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.apache.sling.api.SlingHttpServletRequest;
import org.apache.sling.api.SlingHttpServletResponse;
import org.apache.sling.api.servlets.HttpConstants;
import org.apache.sling.api.servlets.ServletResolverConstants;
import org.apache.sling.api.servlets.SlingSafeMethodsServlet;
import org.osgi.framework.Constants;
import org.osgi.service.component.annotations.Component;
@Component(
service = Servlet.class,
property = {
Constants.SERVICE_DESCRIPTION + "=Search Proxy servlet",
ServletResolverConstants.SLING_SERVLET_METHODS + "=" + HttpConstants.METHOD_GET,
ServletResolverConstants.SLING_SERVLET_RESOURCE_TYPES + "="
+ SearchProxyServlet.RESOURCE_TYPE,
ServletResolverConstants.SLING_SERVLET_EXTENSIONS + "=json",
ServletResolverConstants.SLING_SERVLET_SELECTORS + "=searchproxy"
})
public class SearchProxyServlet extends SlingSafeMethodsServlet {
public static final String RESOURCE_TYPE = "some/resource/type";
@Override
protected void doGet(SlingHttpServletRequest slingRequest, SlingHttpServletResponse slingResponse)
throws ServletException, IOException {
// prepare credentials
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(
new AuthScope("search.com", 443),
new UsernamePasswordCredentials("test", "test"));
slingResponse.setContentType("application/json");
try (CloseableHttpClient httpClient =
HttpClients.custom().setDefaultCredentialsProvider(credentialsProvider).build()) {
HttpGet httpGet = new HttpGet(new URI("https://search.com/endpoint.json"));
try (CloseableHttpResponse httpResponse = httpClient.execute(httpGet)) {
slingResponse.getWriter().write(EntityUtils.toString(httpResponse.getEntity()));
}
} catch (URISyntaxException e) {
e.printStackTrace();
}
}
}

As you can see, we are sending a request to https://search.com/endpoint.json so when you write a unit test for this and invoke doGet, a request will always be sent. But we don’t want that, we want to mock that request. You could use PowerMock, but adding that to your project introduces its own problems. PowerMock is intended for experienced developers and excessive use of it may be an indication of bad implementation/architecture.
A better implementation using an OSGI service

We can move the httpClient to its own OSGI service:
package com.ahmedmusallam.service;
import java.io.IOException;
import java.net.URI;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
@Component(
immediate = true,
property = {"label=Http Client Service", "description=A service for making HTTP calls"},
service = HttpClientService.class)
public class HttpClientService {
/** Perform a get request with the provided credentials provider. */
public String doGet(URI uri, CredentialsProvider credentialsProvider) throws IOException {
if (credentialsProvider == null || uri == null) {
return null;
}
try (CloseableHttpClient httpClient =
HttpClients.custom().setDefaultCredentialsProvider(credentialsProvider).build()) {
HttpGet httpGet = new HttpGet(uri);
try (CloseableHttpResponse httpResponse = httpClient.execute(httpGet)) {
return EntityUtils.toString(httpResponse.getEntity());
}
}
}
}

This makes it easier to mock the service or provide our own implementation of it in our test class.
An improved implementation of the proxy servlet:
// Agian, crude impl to illustrate the point
import com.ahmedmusallam.service.HttpClientService;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import javax.servlet.Servlet;
import javax.servlet.ServletException;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.sling.api.SlingHttpServletRequest;
import org.apache.sling.api.SlingHttpServletResponse;
import org.apache.sling.api.servlets.HttpConstants;
import org.apache.sling.api.servlets.ServletResolverConstants;
import org.apache.sling.api.servlets.SlingSafeMethodsServlet;
import org.osgi.framework.Constants;
import org.osgi.service.component.annotations.Component;
import org.osgi.service.component.annotations.Reference;
@Component(
service = Servlet.class,
property = {
Constants.SERVICE_DESCRIPTION + "=Search Proxy servlet",
ServletResolverConstants.SLING_SERVLET_METHODS + "=" + HttpConstants.METHOD_GET,
ServletResolverConstants.SLING_SERVLET_RESOURCE_TYPES + "="
+ SearchProxyServlet.RESOURCE_TYPE,
ServletResolverConstants.SLING_SERVLET_EXTENSIONS + "=json",
ServletResolverConstants.SLING_SERVLET_SELECTORS + "=searchproxy"
})
public class SearchProxyServlet extends SlingSafeMethodsServlet {
public static final String RESOURCE_TYPE = "some/resource/type";
private HttpClientService httpClientService;
@Reference
public void setHttpClientService(HttpClientService httpClientService) {
this.httpClientService = httpClientService;
}
@Override
protected void doGet(SlingHttpServletRequest slingRequest, SlingHttpServletResponse slingResponse)
throws ServletException, IOException {
// prepare credentials
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(
new AuthScope("test.com", 443),
new UsernamePasswordCredentials("test", "test"));
slingResponse.setContentType("application/json");
try {
String response = httpClientService.doGet(new URI("https://test.com/endpoint.json"), credentialsProvider);
slingResponse.getWriter().write(response);
} catch (URISyntaxException e) {
e.printStackTrace();
}
}
}

Now, this is all good, and we can mock the httpClientService and return a specific string, here is an example test:
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.Mockito.*;
import com.ahmedmusallam.service.HttpClientService;
import com.ahmedmusallam.utils.AppAemContext;
import io.wcm.testing.mock.aem.junit5.AemContext;
import io.wcm.testing.mock.aem.junit5.AemContextExtension;
import java.io.IOException;
import java.net.URI;
import javax.servlet.ServletException;
import org.apache.http.client.CredentialsProvider;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
@ExtendWith({AemContextExtension.class, MockitoExtension.class})
class SearchProxyServletTest {
public final AemContext context = AppAemContext.newAemContext();
@Mock
HttpClientService httpClientService = new HttpClientService();
SearchProxyServlet searchProxyServlet = new SearchProxyServlet();
@BeforeEach
void beforeEach() throws IOException {
when(httpClientService.doGet(any(URI.class), any(CredentialsProvider.class))).thenReturn("{}");
searchProxyServlet.setHttpClientService(httpClientService);
}
@Test
void doGet() throws ServletException, IOException {
// cover case where query
searchProxyServlet.doGet(context.request(), context.response());
assertEquals("{}", context.response().getOutputAsString());
}
}
Testing the HttpClientService

All good so far! But what about testing HttpClientService itself? For that, we would need an HTTP server to run before the test class runs and stop right after. I have found a jUnit4 @Rule for such server here: https://gist.github.com/rponte/710d65dc3beb28d97655. However, I’m using jUnit 5. So I’ve converted that rule into a jUnit5 Extension and here it is:

You can also see it in this gist
import com.sun.net.httpserver.HttpHandler;
import com.sun.net.httpserver.HttpServer;
import java.net.InetSocketAddress;
import java.net.URI;
import java.net.URISyntaxException;
import org.apache.http.client.utils.URIBuilder;
import org.junit.jupiter.api.extension.AfterAllCallback;
import org.junit.jupiter.api.extension.BeforeAllCallback;
import org.junit.jupiter.api.extension.ExtensionContext;
/*
* Note: I chose to implement `BeforeAllCallback` and AfterAllCallback
* but not `AfterEachCallback` and `BeforeEachCallback` for performance reasons.
* I wanted to only run one server per test class and I can register handlers
* on a per-test-method basis. You could implement the `BeforeEachCallback` and `AfterEachCallback`
* interfaces if you really need that behavior.
*/
public class HttpServerExtension implements BeforeAllCallback, AfterAllCallback {
public static final int PORT = 6991;
public static final String HOST = "localhost";
public static final String SCHEME = "http";
private com.sun.net.httpserver.HttpServer server;
@Override
public void afterAll(ExtensionContext extensionContext) throws Exception {
if (server != null) {
server.stop(0); // doesn't wait all current exchange handlers complete
}
}
@Override
public void beforeAll(ExtensionContext extensionContext) throws Exception {
server = HttpServer.create(new InetSocketAddress(PORT), 0);
server.setExecutor(null); // creates a default executor
server.start();
}
public static URI getUriFor(String path) throws URISyntaxException{
return new URIBuilder()
.setScheme(SCHEME)
.setHost(HOST)
.setPort(PORT)
.setPath(path)
.build();
}
public void registerHandler(String uriToHandle, HttpHandler httpHandler) {
server.createContext(uriToHandle, httpHandler);
}
}

As you can see, I run an HTTP server before a test class is run, and stop the server after the test class is run.

and this is the code for a JsonSuccessHandler:

You could, of course, write your own simple handler for other types of requests.
package com.ahmedmusallam.extension;
import java.io.IOException;
import java.nio.charset.Charset;
import org.apache.commons.io.IOUtils;
import java.net.HttpURLConnection;
import com.sun.net.httpserver.HttpExchange;
import com.sun.net.httpserver.HttpHandler;
// credit: https://gist.github.com/rponte/710d65dc3beb28d97655#file-httpserverrule-java
public class JsonSuccessHandler implements HttpHandler {
private String responseBody;
private static final String contentType = "application/json";
public JsonSuccessHandler() {}
public JsonSuccessHandler(String responseBody) {
this.responseBody = responseBody;
}
@Override
public void handle(HttpExchange exchange) throws IOException {
exchange.getResponseHeaders().add("Content-Type", contentType);
exchange.sendResponseHeaders(HttpURLConnection.HTTP_OK, responseBody.length());
IOUtils.write(responseBody, exchange.getResponseBody(), Charset.defaultCharset());
exchange.close();
}
}

Now, let’s write the unit test for our HttpClientService:
package com.ahmedmusallam.service;
import static org.junit.jupiter.api.Assertions.*;
import com.ahmedmusallam.extension.HttpServerExtension;
import com.ahmedmusallam.extension.JsonSuccessHandler;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.api.extension.RegisterExtension;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
@ExtendWith(MockitoExtension.class)
class HttpClientServiceTest {
private HttpClientService httpClientService = new HttpClientService();
@RegisterExtension // MUST be static, see: https://junit.org/junit5/docs/current/user-guide/#extensions-registration-programmatic-static-fields
static HttpServerExtension httpServerExtension = new HttpServerExtension();
@Mock
CredentialsProvider credentialsProvider;
@Test
void doGet() throws IOException, URISyntaxException {
assertNull(httpClientService.doGet(null, credentialsProvider));
assertNull(httpClientService.doGet(new URIBuilder().build(), null));
httpServerExtension.registerHandler("/test", new JsonSuccessHandler("{}"));
URI uri = HttpServerExtension.getUriFor("/test");
assertEquals("{}", httpClientService.doGet(uri, new BasicCredentialsProvider()));
}
}

As you can see, I’ve created an HttpServerExtension and registered a handler for path /test with an expected result, then ran my service’s doGet method against that handler and verified the output.

That’s it! You can add more methods to send POST requests and other types of requests to the HttpClientService and test those in the same fashion.