Breaking: Make rawResponse a Promise<Buffer>

With `debugging protocol` connectors we need to do another request to
get the raw bytes of the response. With this change `sonar` will only
do the extra request when is needed, caching the result for later use.

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

Fix #164
Close #534

src/lib/connectors/debugging-protocol-common/debugging-protocol-connector.ts

@@ -298,13 +298,19 @@ export class Connector implements IConnector {
         }
     }
 
-    private async getResponseBody(cdpResponse) {
+    private async getResponseBody(cdpResponse): Promise<{ content: string, rawContent: Buffer, rawResponse(): Promise<Buffer> }> {
         let content: string = '';
         let rawContent: Buffer = null;
-        let rawResponse: Buffer = null;
+        const rawResponse = (): Promise<Buffer> => {
+            return Promise.resolve(null);
+        };
+        const fetchContent = this.fetchContent;
+
+        const defaultBody = { content, rawContent, rawResponse };
 
         if (cdpResponse.response.status !== 200) {
-            return { content, rawContent, rawResponse };
+            // TODO: is this right? no-friendly-error-pages won't have a problem?
+            return defaultBody;
         }
 
         try {
@@ -314,22 +320,51 @@ export class Connector implements IConnector {
             content = body;
             rawContent = Buffer.from(body, encoding);
 
-            if (rawContent.length.toString() === cdpResponse.response.headers['Content-Length']) {
-                // Response wasn't compressed so both buffers are the same
-                rawResponse = rawContent;
-            } else {
-                rawResponse = null; // TODO: Find a way to get this data
-            }
+            const returnValue = {
+                content,
+                rawContent,
+                rawResponse: () => {
+                    const self = (this as any);
+                    const cached = self._rawResponse;
+
+                    if (cached) {
+                        return Promise.resolve(cached);
+                    }
+
+                    if (rawContent.length.toString() === cdpResponse.response.headers['Content-Length']) {
+                        // Response wasn't compressed so both buffers are the same
+                        return Promise.resolve(rawContent);
+                    }
+
+                    const { url: responseUrl, requestHeaders: headers } = cdpResponse.response;
+
+                    return fetchContent(responseUrl, headers)
+                        .then((result) => {
+                            const { response: { body: { rawResponse: rr } } } = result;
+
+                            return rr();
+                        })
+                        .then((value) => {
+                            self._rawResponse = value;
+
+                            return value;
+                        });
+                }
+            };
+
+            debug(`Content for ${cutString(cdpResponse.response.url)} downloaded`);
+
+            return returnValue;
         } catch (e) {
             debug(`Body requested after connection closed for request ${cdpResponse.requestId}`);
             rawContent = Buffer.alloc(0);
         }
         debug(`Content for ${cutString(cdpResponse.response.url)} downloaded`);
 
-        return { content, rawContent, rawResponse };
+        return defaultBody;
     }
 
-    /** Returns a Response for the given request  */
+    /** Returns a Response for the given request. */
     private async createResponse(cdpResponse): Promise<IResponse> {
         const resourceUrl: string = cdpResponse.response.url;
         const hops: Array<string> = this._redirects.calculate(resourceUrl);
@@ -802,7 +837,8 @@ export class Connector implements IConnector {
     public async fetchContent(target: URL | string, customHeaders?: object): Promise<INetworkData> {
         // TODO: This should create a new tab, navigate to the
         // resource and control what is received somehow via an event.
-        const headers = Object.assign({}, this._headers, customHeaders);
+        const assigns = _.compact([this && this._headers, customHeaders]);
+        const headers = Object.assign({}, ...assigns);
         const href: string = typeof target === 'string' ? target : target.href;
         const request: Requester = new Requester({ headers });
         const response: INetworkData = await request.get(href);

src/lib/connectors/jsdom/jsdom.ts

@@ -111,7 +111,9 @@ class JSDOMConnector implements IConnector {
                     content: body,
                     contentEncoding: null,
                     rawContent: null,
-                    rawResponse: null
+                    rawResponse() {
+                        return Promise.resolve(null);
+                    }
                 },
                 headers: null,
                 hops: [],

src/lib/connectors/utils/requester.ts

@@ -118,7 +118,9 @@ export class Requester {
                             content: body,
                             contentEncoding: charset,
                             rawContent: rawBody,
-                            rawResponse: rawBodyResponse
+                            rawResponse: () => {
+                                return Promise.resolve(rawBodyResponse);
+                            }
                         },
                         headers: response.headers,
                         hops,

src/lib/types/network.ts

@@ -14,7 +14,7 @@ export interface IResponseBody {
     /** The uncompressed bytes of the response's body. */
     rawContent: Buffer;
     /** The original bytes of the body. They could be compressed or not. */
-    rawResponse: Buffer;
+    rawResponse(): Promise<Buffer>;
 }
 
 /** Response data from fetching an item using a connector. */

tests/lib/connectors/events.ts

@@ -290,7 +290,7 @@ const testConnectorEvents = (connectorInfo) => {
         }
 
         // List of events that only have to be called once per execution
-        const singles = ['fetch::error', 'scan::start', 'scan::end', 'manifestfetch::missing'];
+        const singles = ['fetch::error', 'scan::start', 'scan::end', 'manifestfetch::missing', 'targetfetch::start', 'targetfetch::end'];
         const groupedEvents = _.groupBy(invokes, (invoke) => {
             return invoke[0];
         });

tests/lib/connectors/fetchContent.ts

@@ -31,7 +31,7 @@ test.afterEach.always(async (t) => {
     await t.context.connector.close();
 });
 
-const testConnectorEvaluate = (connectorInfo) => {
+const testConnectorFetchContent = (connectorInfo) => {
     const connectorBuilder: IConnectorBuilder = connectorInfo.builder;
     const name: string = connectorInfo.name;
 
@@ -46,15 +46,16 @@ const testConnectorEvaluate = (connectorInfo) => {
         server.configure({ '/edge.png': { content: file } });
 
         const result: INetworkData = await connector.fetchContent(url.parse(`http://localhost:${server.port}/edge.png`));
+        const rawResponse = await result.response.body.rawResponse();
 
         t.is(result.response.statusCode, 200);
         t.true(file.equals(result.response.body.rawContent), 'rawContent is the same');
-        // Because it is an image, the rawResponse should be the same
-        t.true(file.equals(result.response.body.rawResponse), 'rawResponse is the same');
+        // Because it is an image and it is not send compressed, the rawResponse should be the same
+        t.true(file.equals(rawResponse), 'rawResponse is the same');
     });
 
 };
 
 builders.forEach((connector) => {
-    testConnectorEvaluate(connector);
+    testConnectorFetchContent(connector);
 });

tests/lib/connectors/requestResponse.ts

@@ -0,0 +1,162 @@
+/**
+ * @fileoverview Minimum event functionality a connector must implement
+ * in order to be valid.
+ */
+
+/* eslint-disable no-sync */
+
+import * as fs from 'fs';
+import * as path from 'path';
+import * as url from 'url';
+import * as zlib from 'zlib';
+
+import * as _ from 'lodash';
+import * as sinon from 'sinon';
+import test from 'ava';
+
+import { builders } from '../../helpers/connectors';
+import { createServer } from '../../helpers/test-server';
+import { IConnector, IConnectorBuilder } from '../../../src/lib/types';
+
+const sourceHtml = fs.readFileSync(path.join(__dirname, './fixtures/common/index.html'), 'utf8');
+
+/**
+ * Updates all references to localhost to use the right port for the current instance.
+ *
+ * This does a deep search in all the object properties.
+ */
+const updateLocalhost = (content, port) => {
+    if (typeof content === 'string') {
+        return content.replace(/localhost\//g, `localhost:${port}/`);
+    }
+
+    if (typeof content === 'number' || !content) {
+        return content;
+    }
+
+    if (Array.isArray(content)) {
+        const transformed = _.map(content, (value) => {
+            return updateLocalhost(value, port);
+        });
+
+        return transformed;
+    }
+
+    const transformed = _.reduce(content, (obj, value, key) => {
+        obj[key] = updateLocalhost(value, port);
+
+        return obj;
+    }, {});
+
+    return transformed;
+};
+
+
+test.beforeEach(async (t) => {
+    const sonar = {
+        emit() { },
+        emitAsync() { }
+    };
+
+    sinon.spy(sonar, 'emitAsync');
+    sinon.spy(sonar, 'emit');
+
+    const server = createServer();
+
+    await server.start();
+
+    const html = updateLocalhost(sourceHtml, server.port);
+    const gzipHtml = zlib.gzipSync(Buffer.from(html));
+
+    t.context = {
+        gzipHtml,
+        html,
+        server,
+        sonar
+    };
+});
+
+test.afterEach.always(async (t) => {
+    t.context.sonar.emitAsync.restore();
+    t.context.sonar.emit.restore();
+    t.context.server.stop();
+    await t.context.connector.close();
+});
+
+const findEvent = (func, eventName) => {
+    for (let i = 0; i < func.callCount; i++) {
+        const args = func.getCall(i).args;
+
+        if (args[0] === eventName) {
+            return args[1];
+        }
+    }
+
+    return null;
+};
+
+const testRequestResponse = (connectorInfo) => {
+    const connectorBuilder: IConnectorBuilder = connectorInfo.builder;
+    const name: string = connectorInfo.name;
+
+    test(`[${name}] requestResponse`, async (t) => {
+        const { sonar } = t.context;
+        const { emit, emitAsync } = sonar;
+        const connector: IConnector = await (connectorBuilder)(sonar, {});
+        const server = t.context.server;
+
+        t.context.connector = connector;
+
+        server.configure({
+            '/': {
+                content: t.context.gzipHtml,
+                headers: {
+                    'content-encoding': 'gzip',
+                    'content-type': 'text/html'
+                }
+            }
+        });
+
+        await connector.collect(url.parse(`http://localhost:${server.port}/`));
+
+        const invokedTargetFetchEnd = findEvent(emitAsync, 'targetfetch::end') || findEvent(emit, 'targetfetch::end');
+        /* eslint-disable sort-keys */
+        const expectedTargetFetchEnd = {
+            resource: `http://localhost:${server.port}/`,
+            request: { url: `http://localhost:${server.port}/` },
+            response: {
+                body: {
+                    content: t.context.html,
+                    contentEncoding: 'utf-8',
+                    rawContent: Buffer.from(t.context.html),
+                    rawResponse() {
+                        return Promise.resolve(t.context.gzipHtml);
+                    }
+                },
+                hops: [],
+                statusCode: 200,
+                url: 'http://localhost/'
+            }
+        };
+        /* eslint-enable sort-keys */
+
+        if (!invokedTargetFetchEnd) {
+            t.fail(`targetfetch::end' event not found`);
+
+            return;
+        }
+
+        const { body: invokedBody } = invokedTargetFetchEnd.response;
+        const { body: expectedBody } = expectedTargetFetchEnd.response;
+        const [invokedRawResponse, expectedRawResponse] = await Promise.all([invokedBody.rawResponse(), expectedBody.rawResponse()]);
+
+        t.true(expectedRawResponse.equals(invokedRawResponse), 'rawResponses are different');
+        t.true(expectedBody.content === invokedBody.content, 'content is different');
+        t.true(expectedBody.contentEncoding === invokedBody.contentEncoding, 'content-encoding is different');
+        t.true(expectedBody.rawContent.equals(invokedBody.rawContent), 'rawContent is different');
+    });
+};
+
+builders.forEach((connector) => {
+    testRequestResponse(connector);
+});

tests/lib/connectors/utils/requester.ts

@@ -65,7 +65,7 @@ const testTextDecoding = async (t, encoding: string, contentType: string, useCom
     const { requester, server } = t.context;
     const originalBytes = iconv.encode(text, encoding);
     const transformedText = iconv.decode(originalBytes, encoding);
-    const content = useCompression ? await compress(originalBytes) : originalBytes;
+    const content: Buffer = useCompression ? await compress(originalBytes) : originalBytes;
 
     server.configure({
         '/': {
@@ -78,15 +78,16 @@ const testTextDecoding = async (t, encoding: string, contentType: string, useCom
     });
 
     const { response: { body } } = await requester.get(`http://localhost:${server.port}`);
+    const rawResponse = await body.rawResponse();
 
     // body is a `string`
     t.is(body.content, transformedText);
 
     // rawBody is a `Buffer` with the uncompressed bytes of the response
-    t.deepEqual(body.rawContent, originalBytes);
+    t.true(originalBytes.equals(body.rawContent), 'rawContent is not the same');
 
     // rawBodyResponse is a `Buffer` with the original bytes of the response
-    t.deepEqual(body.rawResponse, content);
+    t.true(content.equals(rawResponse));
 };
 
 supportedEncodings.forEach((encoding) => {