From ddf663208e2c017bba6d526140a4fb51362da92d Mon Sep 17 00:00:00 2001 From: Jason Eric Klaes Hoetger Date: Sat, 11 Jul 2015 13:07:06 -0700 Subject: [PATCH 1/2] Added error entries to HAR for failed HTTP requests. Updated to latest bmp-littleproxy build. --- .../filters/BrowserMobHttpFilterChain.java | 14 + .../bmp/filters/HarCaptureFilter.java | 148 ++++++++- .../bmp/ssl/BrowserMobProxyMitmManager.java | 5 +- .../net/lightbody/bmp/proxy/NewHarTest.groovy | 308 +++++++++++++++++- .../lightbody/bmp/core/har/HarResponse.java | 24 +- pom.xml | 2 +- 6 files changed, 480 insertions(+), 21 deletions(-) diff --git a/browsermob-core-littleproxy/src/main/java/net/lightbody/bmp/filters/BrowserMobHttpFilterChain.java b/browsermob-core-littleproxy/src/main/java/net/lightbody/bmp/filters/BrowserMobHttpFilterChain.java index 867b95a7f..25a6500cd 100644 --- a/browsermob-core-littleproxy/src/main/java/net/lightbody/bmp/filters/BrowserMobHttpFilterChain.java +++ b/browsermob-core-littleproxy/src/main/java/net/lightbody/bmp/filters/BrowserMobHttpFilterChain.java @@ -112,6 +112,13 @@ public HttpObject serverToProxyResponse(HttpObject httpObject) { return processedHttpObject; } + @Override + public void serverToProxyResponseTimedOut() { + for (HttpFilters filter : filters) { + filter.serverToProxyResponseTimedOut(); + } + } + @Override public void serverToProxyResponseReceiving() { for (HttpFilters filter : filters) { @@ -135,6 +142,13 @@ public InetSocketAddress proxyToServerResolutionStarted(String resolvingServerHo return overrideAddress; } + @Override + public void proxyToServerResolutionFailed(String hostAndPort) { + for (HttpFilters filter : filters) { + filter.proxyToServerResolutionFailed(hostAndPort); + } + } + @Override public void proxyToServerResolutionSucceeded(String serverHostAndPort, InetSocketAddress resolvedRemoteAddress) { for (HttpFilters filter : filters) { diff --git a/browsermob-core-littleproxy/src/main/java/net/lightbody/bmp/filters/HarCaptureFilter.java b/browsermob-core-littleproxy/src/main/java/net/lightbody/bmp/filters/HarCaptureFilter.java index fd81e0e52..f4a05e315 100644 --- a/browsermob-core-littleproxy/src/main/java/net/lightbody/bmp/filters/HarCaptureFilter.java +++ b/browsermob-core-littleproxy/src/main/java/net/lightbody/bmp/filters/HarCaptureFilter.java @@ -49,6 +49,39 @@ public class HarCaptureFilter extends HttpsAwareFiltersAdapter { private static final Logger log = LoggerFactory.getLogger(HarCaptureFilter.class); + /** + * The HTTP version string in the {@link HarResponse} for failed requests. + */ + private static final String HTTP_VERSION_STRING_FOR_FAILURE = "unknown"; + + /** + * The HTTP status code in the {@link HarResponse} for failed requests. + */ + private static final int HTTP_STATUS_CODE_FOR_FAILURE = 0; + + /** + * The HTTP status text/reason phrase in the {@link HarResponse} for failed requests. + */ + private static final String HTTP_REASON_PHRASE_FOR_FAILURE = ""; + + /** + * The error message that will be populated in the _error field of the {@link HarResponse} due to a name + * lookup failure. + */ + private static final String RESOLUTION_FAILED_ERROR_MESSAGE = "Unable to resolve host: "; + + /** + * The error message that will be populated in the _error field of the {@link HarResponse} due to a + * connection failure. + */ + private static final String CONNECTION_FAILED_ERROR_MESSAGE = "Unable to connect to host"; + + /** + * The error message that will be populated in the _error field of the {@link HarResponse} when the proxy fails to + * receive a response in a timely manner. + */ + private static final String RESPONSE_TIMED_OUT_ERROR_MESSAGE = "Response timed out"; + private final Har har; /** @@ -230,13 +263,13 @@ public HarCaptureFilter(HttpRequest originalRequest, ChannelHandlerContext ctx, // we may need to capture both the request and the response, so set up the request/response filters and delegate to them when // the corresponding filter methods are invoked. to save time and memory, only set up the capturing filters when // we actually need to capture the data. - if (dataToCapture.contains(CaptureType.REQUEST_CONTENT) || dataToCapture.contains(CaptureType.REQUEST_BINARY_CONTENT)) { + if (this.dataToCapture.contains(CaptureType.REQUEST_CONTENT) || this.dataToCapture.contains(CaptureType.REQUEST_BINARY_CONTENT)) { requestCaptureFilter = new ClientRequestCaptureFilter(originalRequest); } else { requestCaptureFilter = null; } - if (dataToCapture.contains(CaptureType.RESPONSE_CONTENT) || dataToCapture.contains(CaptureType.RESPONSE_BINARY_CONTENT)) { + if (this.dataToCapture.contains(CaptureType.RESPONSE_CONTENT) || this.dataToCapture.contains(CaptureType.RESPONSE_BINARY_CONTENT)) { responseCaptureFilter = new ServerResponseCaptureFilter(originalRequest, true); } else { responseCaptureFilter = null; @@ -281,9 +314,9 @@ public HttpResponse clientToProxyRequest(HttpObject httpObject) { captureRequestHeaders(httpRequest); } - // The HTTP CONNECT to the proxy server establishes the SSL connection to the remote server, but the HTTP CONNECT is not recorded in - // a separate HarEntry. Instead, the ssl and connect times are recorded in the first request between the client and remote server - // after the HTTP CONNECT. + // The HTTP CONNECT to the proxy server establishes the SSL connection to the remote server, but the + // HTTP CONNECT is not recorded in a separate HarEntry (except in case of error). Instead, the ssl and + // connect times are recorded in the first request between the client and remote server after the HTTP CONNECT. captureConnectTiming(); } @@ -344,14 +377,39 @@ public HttpObject serverToProxyResponse(HttpObject httpObject) { return super.serverToProxyResponse(httpObject); } + @Override + public void serverToProxyResponseTimedOut() { + if (har == null && !httpConnect) { + return; + } + + // replace any existing HarResponse that was created if the server sent a partial response + HarResponse response = createHarResponseForFailure(); + harEntry.setResponse(response); + + response.setError(RESPONSE_TIMED_OUT_ERROR_MESSAGE); + + + // include this timeout time in the HarTimings object + long timeoutTimestampNanos = System.nanoTime(); + + // if the proxy started to send the request but has not yet finished, we are currently "sending" + if (sendStartedNanos > 0L && sendFinishedNanos == 0L) { + harEntry.getTimings().setSend(timeoutTimestampNanos - sendStartedNanos, TimeUnit.NANOSECONDS); + } + // if the entire request was sent but the proxy has not begun receiving the response, we are currently "waiting" + else if (sendFinishedNanos > 0L && responseReceiveStartedNanos == 0L) { + harEntry.getTimings().setWait(timeoutTimestampNanos - sendFinishedNanos, TimeUnit.NANOSECONDS); + } + // if the proxy has already begun to receive the response, we are currenting "receiving" + else if (responseReceiveStartedNanos > 0L) { + harEntry.getTimings().setReceive(timeoutTimestampNanos - responseReceiveStartedNanos, TimeUnit.NANOSECONDS); + } + } + protected void captureRequestUrl(HttpRequest httpRequest) { - // the HAR spec defines the request.url field as: - // url [string] - Absolute URL of the request (fragments are not included). - // the URI on the httpRequest may only identify the path of the resource, so find the full URL. - // the full URL consists of the scheme + host + port (if non-standard) + path + query params + fragment. - String url = getFullUrl(httpRequest); + HarRequest request = createHarRequestForHttpRequest(httpRequest); - HarRequest request = new HarRequest(httpRequest.getMethod().toString(), url, httpRequest.getProtocolVersion().text()); harEntry.setRequest(request); // capture query parameters. it is safe to assume the query string is UTF-8, since it "should" be in US-ASCII (a subset of UTF-8), @@ -367,11 +425,27 @@ protected void captureRequestUrl(HttpRequest httpRequest) { } catch (IllegalArgumentException e) { // QueryStringDecoder will throw an IllegalArgumentException if it cannot interpret a query string. rather than cause the entire request to // fail by propagating the exception, simply skip the query parameter capture. - // TODO: add error information to custom fields in the HAR - log.info("Could not decode query parameters on URI: " + httpRequest.getUri(), e); + harEntry.setComment("Unable to decode query parameters on URI: " + httpRequest.getUri()); + log.info("Unable to decode query parameters on URI: " + httpRequest.getUri(), e); } } + /** + * Creates a HarRequest object using the method, url, and HTTP version of the specified request. + * + * @param httpRequest HTTP request on which the HarRequest will be based + * @return a new HarRequest object + */ + private HarRequest createHarRequestForHttpRequest(HttpRequest httpRequest) { + // the HAR spec defines the request.url field as: + // url [string] - Absolute URL of the request (fragments are not included). + // the URI on the httpRequest may only identify the path of the resource, so find the full URL. + // the full URL consists of the scheme + host + port (if non-standard) + path + query params + fragment. + String url = getFullUrl(httpRequest); + + return new HarRequest(httpRequest.getMethod().toString(), url, httpRequest.getProtocolVersion().text()); + } + protected void captureUserAgent(HttpRequest httpRequest) { // save the browser and version if it's not yet been set if (har.getLog().getBrowser() == null) { @@ -390,7 +464,7 @@ protected void captureUserAgent(HttpRequest httpRequest) { } protected void captureRequestHeaderSize(HttpRequest httpRequest) { - String requestLine = httpRequest.getMethod().toString() + ' ' + httpRequest.getUri().toString() + ' ' + httpRequest.getProtocolVersion().toString(); + String requestLine = httpRequest.getMethod().toString() + ' ' + httpRequest.getUri() + ' ' + httpRequest.getProtocolVersion().toString(); // +2 => CRLF after status line, +4 => header/data separation long requestHeadersSize = requestLine.length() + 6; @@ -641,6 +715,24 @@ public InetSocketAddress proxyToServerResolutionStarted(String resolvingServerHo return null; } + @Override + public void proxyToServerResolutionFailed(String hostAndPort) { + //TODO: populate values in har for CONNECT requests when resolution fails + if (har == null && !httpConnect) { + return; + } + + HarResponse response = createHarResponseForFailure(); + harEntry.setResponse(response); + + response.setError(RESOLUTION_FAILED_ERROR_MESSAGE + hostAndPort); + + // record the amount of time we attempted to resolve the hostname in the HarTimings object + if (dnsResolutionStartedNanos > 0L) { + harEntry.getTimings().setDns(System.nanoTime() - dnsResolutionStartedNanos, TimeUnit.NANOSECONDS); + } + } + @Override public void proxyToServerResolutionSucceeded(String serverHostAndPort, InetSocketAddress resolvedRemoteAddress) { if (har == null && !httpConnect) { @@ -689,6 +781,24 @@ public void proxyToServerConnectionSSLHandshakeStarted() { this.sslHandshakeStartedNanos = System.nanoTime(); } + @Override + public void proxyToServerConnectionFailed() { + //TODO: populate values in the har when CONNECT requests fail + if (har == null || httpConnect) { + return; + } + + HarResponse response = createHarResponseForFailure(); + harEntry.setResponse(response); + + response.setError(CONNECTION_FAILED_ERROR_MESSAGE); + + // record the amount of time we attempted to connect in the HarTimings object + if (connectionStartedNanos > 0L) { + harEntry.getTimings().setConnect(System.nanoTime() - connectionStartedNanos, TimeUnit.NANOSECONDS); + } + } + @Override public void proxyToServerConnectionSucceeded() { if (har == null && !httpConnect) { @@ -800,4 +910,14 @@ public long getDnsTimeNanos() { } } + /** + * Creates a HarResponse object for failed requests. Normally the HarResponse is populated when the response is received + * from the server, but if the request fails due to a name resolution issue, connection problem, timeout, etc., no + * HarResponse would otherwise be created. + * + * @return a new HarResponse object with invalid HTTP status code (0) and version string ("unknown") + */ + private static HarResponse createHarResponseForFailure() { + return new HarResponse(HTTP_STATUS_CODE_FOR_FAILURE, HTTP_REASON_PHRASE_FOR_FAILURE, HTTP_VERSION_STRING_FOR_FAILURE); + } } diff --git a/browsermob-core-littleproxy/src/main/java/net/lightbody/bmp/ssl/BrowserMobProxyMitmManager.java b/browsermob-core-littleproxy/src/main/java/net/lightbody/bmp/ssl/BrowserMobProxyMitmManager.java index b12ff11fe..99b31bf71 100644 --- a/browsermob-core-littleproxy/src/main/java/net/lightbody/bmp/ssl/BrowserMobProxyMitmManager.java +++ b/browsermob-core-littleproxy/src/main/java/net/lightbody/bmp/ssl/BrowserMobProxyMitmManager.java @@ -4,7 +4,6 @@ import javax.net.ssl.SSLEngine; import javax.net.ssl.SSLSession; -import java.net.InetSocketAddress; /** * This implementation mirrors the implementation of {@link org.littleshoot.proxy.extras.SelfSignedMitmManager}, but uses the @@ -15,8 +14,8 @@ public class BrowserMobProxyMitmManager implements MitmManager { new BrowserMobSslEngineSource(); @Override - public SSLEngine serverSslEngine(InetSocketAddress inetSocketAddress) { - return bmpSslEngineSource.newSslEngine(inetSocketAddress.getHostString(), inetSocketAddress.getPort()); + public SSLEngine serverSslEngine(String host, int port) { + return bmpSslEngineSource.newSslEngine(host, port); } @Override diff --git a/browsermob-core-littleproxy/src/test/groovy/net/lightbody/bmp/proxy/NewHarTest.groovy b/browsermob-core-littleproxy/src/test/groovy/net/lightbody/bmp/proxy/NewHarTest.groovy index fe509885b..84f149c44 100644 --- a/browsermob-core-littleproxy/src/test/groovy/net/lightbody/bmp/proxy/NewHarTest.groovy +++ b/browsermob-core-littleproxy/src/test/groovy/net/lightbody/bmp/proxy/NewHarTest.groovy @@ -8,6 +8,9 @@ import net.lightbody.bmp.core.har.HarContent import net.lightbody.bmp.core.har.HarCookie import net.lightbody.bmp.core.har.HarEntry import net.lightbody.bmp.core.har.HarNameValuePair +import net.lightbody.bmp.core.har.HarResponse +import net.lightbody.bmp.core.har.HarTimings +import net.lightbody.bmp.filters.HarCaptureFilter import net.lightbody.bmp.proxy.dns.AdvancedHostResolver import net.lightbody.bmp.proxy.test.util.MockServerTest import net.lightbody.bmp.proxy.test.util.ProxyServerTest @@ -15,6 +18,7 @@ import net.lightbody.bmp.proxy.util.IOUtils import org.apache.http.client.methods.CloseableHttpResponse import org.apache.http.client.methods.HttpGet import org.junit.After +import org.junit.Ignore import org.junit.Test import org.mockito.invocation.InvocationOnMock import org.mockito.stubbing.Answer @@ -116,7 +120,7 @@ class NewHarTest extends MockServerTest { proxy.newHar() ProxyServerTest.getNewHttpClient(proxy.port).withCloseable { - String responseBody = IOUtils.toStringAndClose(it.execute(new HttpGet("http://localhost:${mockServerPort}/testCaptureResponseCookiesInHar")).getEntity().getContent()); + String responseBody = IOUtils.toStringAndClose(it.execute(new HttpGet("https://localhost:${mockServerPort}/testCaptureResponseCookiesInHar")).getEntity().getContent()); assertEquals("Did not receive expected response from mock server", "success", responseBody); }; @@ -546,5 +550,307 @@ class NewHarTest extends MockServerTest { } + @Test + void testHttpDnsFailureCapturedInHar() { + AdvancedHostResolver mockFailingResolver = mock(AdvancedHostResolver) + when(mockFailingResolver.resolve("www.doesnotexist.address")).thenReturn([]) + + proxy = new BrowserMobProxyServer(); + proxy.setHostNameResolver(mockFailingResolver) + proxy.start() + + proxy.newHar() + + String requestUrl = "http://www.doesnotexist.address/some-resource" + + ProxyServerTest.getNewHttpClient(proxy.port).withCloseable { + CloseableHttpResponse response = it.execute(new HttpGet(requestUrl)) + assertEquals("Did not receive HTTP 502 from proxy", 502, response.getStatusLine().getStatusCode()) + }; + + Thread.sleep(500) + Har har = proxy.getHar() + + assertThat("Expected to find entries in the HAR", har.getLog().getEntries(), not(empty())) + + // make sure request data is still captured despite the failure + String capturedUrl = har.log.entries[0].request.url + assertEquals("URL captured in HAR did not match request URL", requestUrl, capturedUrl) + + HarResponse harResponse = har.log.entries[0].response + assertNotNull("No HAR response found", harResponse) + + assertEquals("Error in HAR response did not match expected DNS failure error message", HarCaptureFilter.RESOLUTION_FAILED_ERROR_MESSAGE + "www.doesnotexist.address", harResponse.error) + assertEquals("Expected HTTP status code of 0 for failed request", HarCaptureFilter.HTTP_STATUS_CODE_FOR_FAILURE, harResponse.status) + assertEquals("Expected unknown HTTP version for failed request", HarCaptureFilter.HTTP_VERSION_STRING_FOR_FAILURE, harResponse.httpVersion) + assertEquals("Expected default value for headersSize for failed request", -1L, harResponse.headersSize) + assertEquals("Expected default value for bodySize for failed request", -1L, harResponse.bodySize) + + HarTimings harTimings = har.log.entries[0].timings + assertNotNull("No HAR timings found", harTimings) + + assertThat("Expected dns time to be populated after dns resolution failure", harTimings.getDns(TimeUnit.NANOSECONDS), greaterThan(0L)) + + assertEquals("Expected HAR timings to contain default values after DNS failure", -1L, harTimings.getConnect(TimeUnit.NANOSECONDS)) + assertEquals("Expected HAR timings to contain default values after DNS failure", -1L, harTimings.getSsl(TimeUnit.NANOSECONDS)) + assertEquals("Expected HAR timings to contain default values after DNS failure", 0L, harTimings.getSend(TimeUnit.NANOSECONDS)) + assertEquals("Expected HAR timings to contain default values after DNS failure", 0L, harTimings.getWait(TimeUnit.NANOSECONDS)) + assertEquals("Expected HAR timings to contain default values after DNS failure", 0L, harTimings.getReceive(TimeUnit.NANOSECONDS)) + } + + // TODO: unignore when a strategy for handling failed HTTP CONNECT requests is implemented + @Ignore + @Test + void testHttpsDnsFailureCapturedInHar() { + AdvancedHostResolver mockFailingResolver = mock(AdvancedHostResolver) + when(mockFailingResolver.resolve("www.doesnotexist.address")).thenReturn([]) + + proxy = new BrowserMobProxyServer(); + proxy.setHostNameResolver(mockFailingResolver) + proxy.start() + + proxy.newHar() + + String requestUrl = "https://www.doesnotexist.address/some-resource" + + ProxyServerTest.getNewHttpClient(proxy.port).withCloseable { + CloseableHttpResponse response = it.execute(new HttpGet(requestUrl)) + assertEquals("Did not receive HTTP 502 from proxy", 502, response.getStatusLine().getStatusCode()) + }; + + Thread.sleep(500) + Har har = proxy.getHar() + + assertThat("Expected to find entries in the HAR", har.getLog().getEntries(), not(empty())) + + // make sure request data is still captured despite the failure + String capturedUrl = har.log.entries[0].request.url + assertEquals("URL captured in HAR did not match request URL", requestUrl, capturedUrl) + + HarResponse harResponse = har.log.entries[0].response + assertNotNull("No HAR response found", harResponse) + + assertEquals("Error in HAR response did not match expected DNS failure error message", HarCaptureFilter.RESOLUTION_FAILED_ERROR_MESSAGE + "www.doesnotexist.address", harResponse.error) + assertEquals("Expected HTTP status code of 0 for failed request", HarCaptureFilter.HTTP_STATUS_CODE_FOR_FAILURE, harResponse.status) + assertEquals("Expected unknown HTTP version for failed request", HarCaptureFilter.HTTP_VERSION_STRING_FOR_FAILURE, harResponse.httpVersion) + assertEquals("Expected default value for headersSize for failed request", -1L, harResponse.headersSize) + assertEquals("Expected default value for bodySize for failed request", -1L, harResponse.bodySize) + + HarTimings harTimings = har.log.entries[0].timings + assertNotNull("No HAR timings found", harTimings) + + assertThat("Expected dns time to be populated after dns resolution failure", harTimings.getDns(TimeUnit.NANOSECONDS), greaterThan(0L)) + + assertEquals("Expected HAR timings to contain default values after DNS failure", -1L, harTimings.getConnect(TimeUnit.NANOSECONDS)) + assertEquals("Expected HAR timings to contain default values after DNS failure", -1L, harTimings.getSsl(TimeUnit.NANOSECONDS)) + assertEquals("Expected HAR timings to contain default values after DNS failure", 0L, harTimings.getSend(TimeUnit.NANOSECONDS)) + assertEquals("Expected HAR timings to contain default values after DNS failure", 0L, harTimings.getWait(TimeUnit.NANOSECONDS)) + assertEquals("Expected HAR timings to contain default values after DNS failure", 0L, harTimings.getReceive(TimeUnit.NANOSECONDS)) + } + + @Test + void testHttpConnectTimeoutCapturedInHar() { + proxy = new BrowserMobProxyServer(); + proxy.start() + + proxy.newHar() + + String requestUrl = "http://localhost:0/some-resource" + + ProxyServerTest.getNewHttpClient(proxy.port).withCloseable { + CloseableHttpResponse response = it.execute(new HttpGet(requestUrl)) + assertEquals("Did not receive HTTP 502 from proxy", 502, response.getStatusLine().getStatusCode()) + }; + + Thread.sleep(500) + Har har = proxy.getHar() + + assertThat("Expected to find entries in the HAR", har.getLog().getEntries(), not(empty())) + + // make sure request data is still captured despite the failure + String capturedUrl = har.log.entries[0].request.url + assertEquals("URL captured in HAR did not match request URL", requestUrl, capturedUrl) + + HarResponse harResponse = har.log.entries[0].response + assertNotNull("No HAR response found", harResponse) + + assertEquals("Error in HAR response did not match expected connection failure error message", HarCaptureFilter.CONNECTION_FAILED_ERROR_MESSAGE, harResponse.error) + assertEquals("Expected HTTP status code of 0 for failed request", HarCaptureFilter.HTTP_STATUS_CODE_FOR_FAILURE, harResponse.status) + assertEquals("Expected unknown HTTP version for failed request", HarCaptureFilter.HTTP_VERSION_STRING_FOR_FAILURE, harResponse.httpVersion) + assertEquals("Expected default value for headersSize for failed request", -1L, harResponse.headersSize) + assertEquals("Expected default value for bodySize for failed request", -1L, harResponse.bodySize) + + HarTimings harTimings = har.log.entries[0].timings + assertNotNull("No HAR timings found", harTimings) + + assertThat("Expected dns time to be populated after connection failure", harTimings.getDns(TimeUnit.NANOSECONDS), greaterThan(0L)) + assertThat("Expected connect time to be populated after connection failure", harTimings.getConnect(TimeUnit.NANOSECONDS), greaterThan(0L)) + assertEquals("Expected HAR timings to contain default values after connection failure", -1L, harTimings.getSsl(TimeUnit.NANOSECONDS)) + assertEquals("Expected HAR timings to contain default values after connection failure", 0L, harTimings.getSend(TimeUnit.NANOSECONDS)) + assertEquals("Expected HAR timings to contain default values after connection failure", 0L, harTimings.getWait(TimeUnit.NANOSECONDS)) + assertEquals("Expected HAR timings to contain default values after connection failure", 0L, harTimings.getReceive(TimeUnit.NANOSECONDS)) + } + + // TODO: unignore when a strategy for handling failed HTTP CONNECT requests is implemented + @Ignore + @Test + void testHttpsConnectTimeoutCapturedInHar() { + proxy = new BrowserMobProxyServer(); + proxy.start() + + proxy.newHar() + + String requestUrl = "https://localhost:0/some-resource" + + ProxyServerTest.getNewHttpClient(proxy.port).withCloseable { + CloseableHttpResponse response = it.execute(new HttpGet(requestUrl)) + assertEquals("Did not receive HTTP 502 from proxy", 502, response.getStatusLine().getStatusCode()) + }; + + Thread.sleep(500) + Har har = proxy.getHar() + + assertThat("Expected to find entries in the HAR", har.getLog().getEntries(), not(empty())) + + // make sure request data is still captured despite the failure + String capturedUrl = har.log.entries[0].request.url + assertEquals("URL captured in HAR did not match request URL", requestUrl, capturedUrl) + + HarResponse harResponse = har.log.entries[0].response + assertNotNull("No HAR response found", harResponse) + + assertEquals("Error in HAR response did not match expected connection failure error message", HarCaptureFilter.CONNECTION_FAILED_ERROR_MESSAGE, harResponse.error) + assertEquals("Expected HTTP status code of 0 for failed request", HarCaptureFilter.HTTP_STATUS_CODE_FOR_FAILURE, harResponse.status) + assertEquals("Expected unknown HTTP version for failed request", HarCaptureFilter.HTTP_VERSION_STRING_FOR_FAILURE, harResponse.httpVersion) + assertEquals("Expected default value for headersSize for failed request", -1L, harResponse.headersSize) + assertEquals("Expected default value for bodySize for failed request", -1L, harResponse.bodySize) + + HarTimings harTimings = har.log.entries[0].timings + assertNotNull("No HAR timings found", harTimings) + + assertThat("Expected dns time to be populated after connection failure", harTimings.getDns(TimeUnit.NANOSECONDS), greaterThan(0L)) + assertThat("Expected connect time to be populated after connection failure", harTimings.getConnect(TimeUnit.NANOSECONDS), greaterThan(0L)) + assertEquals("Expected HAR timings to contain default values after connection failure", -1L, harTimings.getSsl(TimeUnit.NANOSECONDS)) + assertEquals("Expected HAR timings to contain default values after connection failure", 0L, harTimings.getSend(TimeUnit.NANOSECONDS)) + assertEquals("Expected HAR timings to contain default values after connection failure", 0L, harTimings.getWait(TimeUnit.NANOSECONDS)) + assertEquals("Expected HAR timings to contain default values after connection failure", 0L, harTimings.getReceive(TimeUnit.NANOSECONDS)) + } + + @Test + void testHttpResponseTimeoutCapturedInHar() { + mockServer.when(request() + .withMethod("GET") + .withPath("/testResponseTimeoutCapturedInHar"), + Times.once()) + .respond(response() + .withStatusCode(200) + .withDelay(TimeUnit.SECONDS, 10) + .withBody("success")) + + proxy = new BrowserMobProxyServer(); + proxy.setIdleConnectionTimeout(3, TimeUnit.SECONDS) + proxy.start() + + proxy.newHar() + + String requestUrl = "http://localhost:${mockServerPort}/testResponseTimeoutCapturedInHar" + + ProxyServerTest.getNewHttpClient(proxy.port).withCloseable { + CloseableHttpResponse response = it.execute(new HttpGet(requestUrl)) + assertEquals("Did not receive HTTP 504 from proxy", 504, response.getStatusLine().getStatusCode()) + }; + + Thread.sleep(500) + Har har = proxy.getHar() + + assertThat("Expected to find entries in the HAR", har.getLog().getEntries(), not(empty())) + + // make sure request data is still captured despite the failure + String capturedUrl = har.log.entries[0].request.url + assertEquals("URL captured in HAR did not match request URL", requestUrl, capturedUrl) + + HarResponse harResponse = har.log.entries[0].response + assertNotNull("No HAR response found", harResponse) + + assertEquals("Error in HAR response did not match expected response timeout error message", HarCaptureFilter.RESPONSE_TIMED_OUT_ERROR_MESSAGE, harResponse.error) + assertEquals("Expected HTTP status code of 0 for response timeout", HarCaptureFilter.HTTP_STATUS_CODE_FOR_FAILURE, harResponse.status) + assertEquals("Expected unknown HTTP version for response timeout", HarCaptureFilter.HTTP_VERSION_STRING_FOR_FAILURE, harResponse.httpVersion) + assertEquals("Expected default value for headersSize for response timeout", -1L, harResponse.headersSize) + assertEquals("Expected default value for bodySize for response timeout", -1L, harResponse.bodySize) + + HarTimings harTimings = har.log.entries[0].timings + assertNotNull("No HAR timings found", harTimings) + + assertEquals("Expected ssl timing to contain default value", -1L, harTimings.getSsl(TimeUnit.NANOSECONDS)) + + // this timeout was caused by a failure of the server to respond, so dns, connect, send, and wait should all be populated, + // but receive should not be populated since no response was received. + assertThat("Expected dns time to be populated", harTimings.getDns(TimeUnit.NANOSECONDS), greaterThan(0L)) + assertThat("Expected connect time to be populated", harTimings.getConnect(TimeUnit.NANOSECONDS), greaterThan(0L)) + assertThat("Expected send time to be populated", harTimings.getSend(TimeUnit.NANOSECONDS), greaterThan(0L)) + assertThat("Expected wait time to be populated", harTimings.getWait(TimeUnit.NANOSECONDS), greaterThan(0L)) + + assertEquals("Expected receive time to not be populated", 0L, harTimings.getReceive(TimeUnit.NANOSECONDS)) + } + + // TODO: unignore when a strategy for handling failed HTTP CONNECT requests is implemented + @Ignore + @Test + void testHttpsResponseTimeoutCapturedInHar() { + mockServer.when(request() + .withMethod("GET") + .withPath("/testResponseTimeoutCapturedInHar"), + Times.once()) + .respond(response() + .withStatusCode(200) + .withDelay(TimeUnit.SECONDS, 10) + .withBody("success")) + + proxy = new BrowserMobProxyServer(); + proxy.setIdleConnectionTimeout(3, TimeUnit.SECONDS) + proxy.start() + + proxy.newHar() + + String requestUrl = "https://localhost:${mockServerPort}/testResponseTimeoutCapturedInHar" + + ProxyServerTest.getNewHttpClient(proxy.port).withCloseable { + CloseableHttpResponse response = it.execute(new HttpGet(requestUrl)) + assertEquals("Did not receive HTTP 504 from proxy", 504, response.getStatusLine().getStatusCode()) + }; + + Thread.sleep(500) + Har har = proxy.getHar() + + assertThat("Expected to find entries in the HAR", har.getLog().getEntries(), not(empty())) + + // make sure request data is still captured despite the failure + String capturedUrl = har.log.entries[0].request.url + assertEquals("URL captured in HAR did not match request URL", requestUrl, capturedUrl) + + HarResponse harResponse = har.log.entries[0].response + assertNotNull("No HAR response found", harResponse) + + assertEquals("Error in HAR response did not match expected response timeout error message", HarCaptureFilter.RESPONSE_TIMED_OUT_ERROR_MESSAGE, harResponse.error) + assertEquals("Expected HTTP status code of 0 for response timeout", HarCaptureFilter.HTTP_STATUS_CODE_FOR_FAILURE, harResponse.status) + assertEquals("Expected unknown HTTP version for response timeout", HarCaptureFilter.HTTP_VERSION_STRING_FOR_FAILURE, harResponse.httpVersion) + assertEquals("Expected default value for headersSize for response timeout", -1L, harResponse.headersSize) + assertEquals("Expected default value for bodySize for response timeout", -1L, harResponse.bodySize) + + HarTimings harTimings = har.log.entries[0].timings + assertNotNull("No HAR timings found", harTimings) + + assertEquals("Expected ssl timing to contain default value", -1L, harTimings.getSsl(TimeUnit.NANOSECONDS)) + + // this timeout was caused by a failure of the server to respond, so dns, connect, send, and wait should all be populated, + // but receive should not be populated since no response was received. + assertThat("Expected dns time to be populated", harTimings.getDns(TimeUnit.NANOSECONDS), greaterThan(0L)) + assertThat("Expected connect time to be populated", harTimings.getConnect(TimeUnit.NANOSECONDS), greaterThan(0L)) + assertThat("Expected send time to be populated", harTimings.getSend(TimeUnit.NANOSECONDS), greaterThan(0L)) + assertThat("Expected wait time to be populated", harTimings.getWait(TimeUnit.NANOSECONDS), greaterThan(0L)) + + assertEquals("Expected receive time to not be populated", 0L, harTimings.getReceive(TimeUnit.NANOSECONDS)) + } + //TODO: Add Request Capture Type tests } diff --git a/browsermob-core/src/main/java/net/lightbody/bmp/core/har/HarResponse.java b/browsermob-core/src/main/java/net/lightbody/bmp/core/har/HarResponse.java index 6470988d0..f82f248dd 100644 --- a/browsermob-core/src/main/java/net/lightbody/bmp/core/har/HarResponse.java +++ b/browsermob-core/src/main/java/net/lightbody/bmp/core/har/HarResponse.java @@ -1,6 +1,7 @@ package net.lightbody.bmp.core.har; import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; @@ -14,10 +15,21 @@ public class HarResponse { private final List headers = new CopyOnWriteArrayList(); private final HarContent content = new HarContent(); private volatile String redirectURL = ""; - private volatile long headersSize; - private volatile long bodySize; + + /* the values of headersSize and bodySize are set to -1 by default, in accordance with the HAR spec: + headersSize [number] - Total number of bytes from the start of the HTTP request message until (and including) the double CRLF before the body. Set to -1 if the info is not available. + bodySize [number] - Size of the request body (POST data payload) in bytes. Set to -1 if the info is not available. + */ + private volatile long headersSize = -1; + private volatile long bodySize = -1; private volatile String comment = ""; + /** + * A custom field indicating that an error occurred, such as DNS resolution failure. + */ + @JsonProperty("_error") + private volatile String error; + public HarResponse() { } @@ -94,4 +106,12 @@ public String getComment() { public void setComment(String comment) { this.comment = comment; } + + public String getError() { + return error; + } + + public void setError(String error) { + this.error = error; + } } diff --git a/pom.xml b/pom.xml index fca25666c..bcac5f131 100644 --- a/pom.xml +++ b/pom.xml @@ -261,7 +261,7 @@ net.lightbody.bmp littleproxy - 1.1.0-beta-bmp-5 + 1.1.0-beta-bmp-6 From bbd44d951cbdfb867824324175f3f1373903b6e3 Mon Sep 17 00:00:00 2001 From: Jason Eric Klaes Hoetger Date: Sat, 11 Jul 2015 13:29:04 -0700 Subject: [PATCH 2/2] Updated to latest selenium. Temporarily ignoring BrowserTests until a solution to the Firefox untrusted certs issue is found. --- .../net/lightbody/bmp/proxy/BrowserTest.java | 17 ++++++++++++----- pom.xml | 2 +- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/browsermob-core/src/test/java/net/lightbody/bmp/proxy/BrowserTest.java b/browsermob-core/src/test/java/net/lightbody/bmp/proxy/BrowserTest.java index 93b58a520..603e6d1ff 100644 --- a/browsermob-core/src/test/java/net/lightbody/bmp/proxy/BrowserTest.java +++ b/browsermob-core/src/test/java/net/lightbody/bmp/proxy/BrowserTest.java @@ -3,8 +3,8 @@ import net.lightbody.bmp.core.har.Har; import net.lightbody.bmp.core.har.HarEntry; import net.lightbody.bmp.proxy.test.util.ProxyServerTest; -import org.junit.Assert; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.openqa.selenium.Proxy; import org.openqa.selenium.WebDriver; @@ -13,11 +13,17 @@ import org.openqa.selenium.remote.CapabilityType; import org.openqa.selenium.remote.DesiredCapabilities; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.not; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; import static org.junit.Assume.assumeFalse; /** * Tests which require a web browser should be placed in this class so they can be properly configured/ignored for CI builds. */ +// TODO: temporarily ignoring because firefox is no longer ignoring untrusted certificates, even when instructed to do so +@Ignore public class BrowserTest extends ProxyServerTest { @Before public void skipForTravisCi() { @@ -47,8 +53,10 @@ public void testCaptureHarHttpsPageWithFirefox() throws Exception { // get the HAR data Har har = proxy.getHar(); + Thread.sleep(500); + // make sure something came back in the har - Assert.assertTrue(!har.getLog().getEntries().isEmpty()); + assertThat("Did not find any entries in the HAR", har.getLog().getEntries(), not(empty())); // show that we can capture the HTML of the root page // NOTE: firefox seems to occasionally make its first request to some mozilla address, so we can't rely on getEntries().get(0) to get the actual Google page @@ -63,7 +71,7 @@ public void testCaptureHarHttpsPageWithFirefox() throws Exception { } } - Assert.assertTrue("Did not find any HAR entry containing the text Google", foundGooglePage); + assertTrue("Did not find any HAR entry containing the text Google", foundGooglePage); } finally { if (driver != null) { driver.quit(); @@ -90,8 +98,7 @@ public void testProxyConfigurationThroughFirefoxProfile() { capabilities.setCapability(CapabilityType.ACCEPT_SSL_CERTS, true); capabilities.setCapability(FirefoxDriver.PROFILE, profile); - capabilities.setCapability(CapabilityType.PROXY, - proxy.seleniumProxy()); + capabilities.setCapability(CapabilityType.PROXY, proxy.seleniumProxy()); driver = new FirefoxDriver(capabilities); driver.get("https://www.gmail.com/"); diff --git a/pom.xml b/pom.xml index bcac5f131..58bbd130a 100644 --- a/pom.xml +++ b/pom.xml @@ -56,7 +56,7 @@ UTF-8 1.7.12 - 2.45.0 + 2.46.0 2.4.4