Add preliminary version of message merging

This fixes #1137 [1].

[1] #1137

CHANGELOG.md

@@ -22,6 +22,10 @@ Other changes:
 
 - The above-mentioned `onAfterStep` hook, is now invoked with a bunch of relevant data, relates to [#1089](https://github.com/badeball/cypress-cucumber-preprocessor/issues/1089).
 
+- Add a tool for [merging messages reports](docs/merging-reports.md), fixes [#1137](https://github.com/badeball/cypress-cucumber-preprocessor/issues/1137).
+
+  - This is first and foremost created to support merging related reports obtained through parallelization using Cypress Cloud.
+
 ## v19.2.0
 
 - Add order option to all hooks, fixes [#481](https://github.com/badeball/cypress-cucumber-preprocessor/issues/481).

docs/merging-reports.md

@@ -0,0 +1,57 @@
+[← Back to documentation](readme.md)
+
+# Parallelization using Cypress Cloud & merging reports
+
+[Parallelization](https://docs.cypress.io/guides/cloud/smart-orchestration/parallelization) using Cypress Cloud is where you run `cypress run --record --parallel` on N machine in order to improve execution time. This typically generates N reports. These are obviously related to each other. They are unique sets of spec results, with no overlap, obtained through sharding (weighing and distribution by Cypress Cloud).
+
+Different messages reports can be combined into a single messages report using the following executable. This behaves like `cat`, it reads file arguments and outputs a combined report to stdout.
+
+```
+$ npx cucumber-merge-messages *.ndsjon
+```
+
+Alternatively you can redirect the output to a new file.
+
+```
+$ npx cucumber-merge-messages *.ndsjon > combined-cucumber-messages.ndjson
+```
+
+Only *messages reports* can be merged. JSON and HTML reports are both products of messages and if you require either, then you can use the [JSON formatter](json-formatter.md) or [HTML formatter](html-formatter.md) to create one from the other, like shown below.
+
+```
+$ npx cucumber-merge-messages *.ndsjon > combined-cucumber-messages.ndjson
+$ npx cucumber-json-formatter < combined-cucumber-messages.ndjson > cucumber-report.json
+$ npx cucumber-html-formatter < combined-cucumber-messages.ndjson > cucumber-report.html
+```
+
+## How do I collect reports created on N different machines?
+
+You need to consult your CI provider's documentation. Below is a very simple example illustrating the use of *artifacts* on Gitlab CI/CD to solve this. The reason for specifying `messagesOutput` is because on Gitlab, [artifacts with the same name from different jobs overwrite each other](https://gitlab.com/gitlab-org/gitlab/-/issues/244714).
+
+```yaml
+stages:
+  - test
+  - report
+
+cypress:
+  stage: test
+  parallel: 5
+  script:
+    - npx cypress run --record --parallel --env messagesOutput=cucumber-messages-$CI_NODE_INDEX.ndjson
+  artifacts:
+    paths:
+      - *.ndjson
+
+combine:
+  stage: report
+  script:
+    - npx cucumber-merge-messages *.ndsjon > cucumber-messages.ndjson
+    - npx cucumber-json-formatter < cucumber-messages.ndjson > cucumber-report.json
+    - npx cucumber-html-formatter < cucumber-messages.ndjson > cucumber-report.html
+  artifacts:
+    paths:
+      - cucumber-messages.ndjson
+      - cucumber-report.json
+      - cucumber-report.html
+```
+

docs/readme.md

@@ -18,6 +18,7 @@
   * [Diagnostics / dry run](diagnostics.md)
   * [JSON formatter](json-formatter.md)
   * [HTML formatter](html-formatter.md)
+  * [Parallelization using Cypress Cloud & merging reports](merging-reports.md)
 * [Mixing Cucumber and non-Cucumber specs](mixing-types.md)
 * [:warning: On event handlers](event-handlers.md)
 * [Frequently asked questions](faq.md)

features/fixtures/another-passed-example.ndjson

@@ -0,0 +1,12 @@
+{"meta":"meta"}
+{"testRunStarted":{"timestamp":{"seconds":0,"nanos":0}}}
+{"source":{"data":"Feature: another feature\n  Scenario: another scenario\n    Given a step","uri":"cypress/e2e/b.feature","mediaType":"text/x.cucumber.gherkin+plain"}}
+{"gherkinDocument":{"feature":{"tags":[],"location":{"line":1,"column":1},"language":"en","keyword":"Feature","name":"another feature","description":"","children":[{"scenario":{"id":"id","tags":[],"location":{"line":2,"column":3},"keyword":"Scenario","name":"another scenario","description":"","steps":[{"id":"id","location":{"line":3,"column":5},"keyword":"Given ","keywordType":"Context","text":"a step"}],"examples":[]}}]},"comments":[],"uri":"cypress/e2e/b.feature"}}
+{"pickle":{"id":"id","uri":"cypress/e2e/b.feature","astNodeIds":["id"],"tags":[],"name":"another scenario","language":"en","steps":[{"id":"id","text":"a step","type":"Context","astNodeIds":["id"]}]}}
+{"stepDefinition":{"id":"id","pattern":{"type":"CUCUMBER_EXPRESSION","source":"a step"},"sourceReference":{"uri":"not available","location":{"line":0}}}}
+{"testCase":{"id":"id","pickleId":"id","testSteps":[{"id":"id","pickleStepId":"id","stepDefinitionIds":["id"]}]}}
+{"testCaseStarted":{"id":"id","testCaseId":"id","attempt":0,"timestamp":{"seconds":0,"nanos":0}}}
+{"testStepStarted":{"testStepId":"id","testCaseStartedId":"id","timestamp":{"seconds":0,"nanos":0}}}
+{"testStepFinished":{"testStepId":"id","testCaseStartedId":"id","testStepResult":{"status":"PASSED","duration":0},"timestamp":{"seconds":0,"nanos":0}}}
+{"testCaseFinished":{"testCaseStartedId":"id","timestamp":{"seconds":0,"nanos":0},"willBeRetried":false}}
+{"testRunFinished":{"timestamp":{"seconds":0,"nanos":0}}}

features/merge_messages.feature

@@ -0,0 +1,32 @@
+Feature: merging reports
+
+  Scenario: two features
+    Given additional preprocessor configuration
+      """
+      {
+        "messages": {
+          "enabled": true
+        }
+      }
+      """
+    And a file named "cypress/e2e/a.feature" with:
+      """
+      Feature: a feature
+        Scenario: a scenario
+          Given a step
+      """
+    And a file named "cypress/e2e/b.feature" with:
+      """
+      Feature: another feature
+        Scenario: another scenario
+          Given a step
+      """
+    And a file named "cypress/support/step_definitions/steps.js" with:
+      """
+      const { Given } = require("@badeball/cypress-cucumber-preprocessor");
+      Given("a step", function() {})
+      """
+    When I run cypress with "--spec cypress/e2e/a.feature --env messagesOutput=cucumber-messages-a.ndjson" (expecting exit code 0)
+    And I run cypress with "--spec cypress/e2e/b.feature --env messagesOutput=cucumber-messages-b.ndjson" (expecting exit code 0)
+    And I merge the messages reports
+    Then there should be a messages similar to "fixtures/multiple-features.ndjson"

features/step_definitions/cli_steps.ts

@@ -1,4 +1,6 @@
+import fs from "fs/promises";
 import util from "util";
+import path from "path";
 import { When, Then } from "@cucumber/cucumber";
 import assert from "assert";
 import childProcess from "child_process";
@@ -7,19 +9,40 @@ import ICustomWorld from "../support/ICustomWorld";
 import { assertAndReturn } from "../support/helpers";
 
 function execAsync(
+  world: ICustomWorld,
   command: string
 ): Promise<{ stdout: string; stderr: string }> {
   return new Promise((resolve, reject) => {
-    childProcess.exec(command, (error, stdout, stderr) => {
-      if (error) {
-        reject(error);
-      } else {
-        resolve({ stdout, stderr });
+    childProcess.exec(
+      command,
+      { cwd: world.tmpDir },
+      (error, stdout, stderr) => {
+        if (error) {
+          reject(error);
+        } else {
+          resolve({ stdout, stderr });
+        }
       }
-    });
+    );
   });
 }
 
+async function parseArgs(
+  world: ICustomWorld,
+  unparsedArgs: string
+): Promise<string[]> {
+  // Use user's preferred shell to split args.
+  const { stdout } = await execAsync(
+    world,
+    `node -p "JSON.stringify(process.argv)" -- ${unparsedArgs}`
+  );
+
+  // Drop 1st arg, which is the path of node.
+  const [, ...extraArgs] = JSON.parse(stdout);
+
+  return extraArgs;
+}
+
 When(
   "I run cypress",
   { timeout: 60 * 1000 },
@@ -32,15 +55,28 @@ When(
   "I run cypress with {string}",
   { timeout: 60 * 1000 },
   async function (this: ICustomWorld, unparsedArgs) {
+    await this.runCypress({ extraArgs: await parseArgs(this, unparsedArgs) });
+  }
+);
+
+When(
+  "I run cypress with {string} \\(expecting exit code {int})",
+  { timeout: 60 * 1000 },
+  async function (
+    this: ICustomWorld,
+    unparsedArgs: string,
+    expectedExitCode: number
+  ) {
     // Use user's preferred shell to split args.
     const { stdout } = await execAsync(
+      this,
       `node -p "JSON.stringify(process.argv)" -- ${unparsedArgs}`
     );
 
     // Drop 1st arg, which is the path of node.
     const [, ...extraArgs] = JSON.parse(stdout);
 
-    await this.runCypress({ extraArgs });
+    await this.runCypress({ extraArgs, expectedExitCode });
   }
 );
 
@@ -60,6 +96,23 @@ When(
   }
 );
 
+When(
+  "I merge the messages reports",
+  { timeout: 60 * 1000 },
+  async function (this: ICustomWorld) {
+    const extraArgs = await parseArgs(this, "*.ndjson");
+
+    await this.runMergeMessages({
+      extraArgs,
+      expectedExitCode: 0,
+    });
+
+    const absoluteFilePath = path.join(this.tmpDir, "cucumber-messages.ndjson");
+
+    await fs.writeFile(absoluteFilePath, expectLastRun(this).output);
+  }
+);
+
 const expectLastRun = (world: ICustomWorld) =>
   assertAndReturn(world.lastRun, "Expected to find information about last run");
 

features/step_definitions/messages_steps.ts

@@ -5,100 +5,14 @@ import { promises as fs } from "fs";
 import assert from "assert";
 import { toByteArray } from "base64-js";
 import { PNG } from "pngjs";
-import { assertAndReturn } from "../support/helpers";
+import {
+  assertAndReturn,
+  ndJsonToString,
+  prepareMessagesReport,
+  stringToNdJson,
+} from "../support/helpers";
 import ICustomWorld from "../support/ICustomWorld";
 
-function isObject(object: any): object is object {
-  return typeof object === "object" && object != null;
-}
-
-// eslint-disable-next-line @typescript-eslint/ban-types
-function hasOwnProperty<X extends {}, Y extends PropertyKey>(
-  obj: X,
-  prop: Y
-): obj is X & Record<Y, unknown> {
-  return Object.prototype.hasOwnProperty.call(obj, prop);
-}
-
-function* traverseTree(object: any): Generator<object, void, any> {
-  if (!isObject(object)) {
-    throw new Error(`Expected object, got ${typeof object}`);
-  }
-
-  yield object;
-
-  for (const property of Object.values(object)) {
-    if (isObject(property)) {
-      yield* traverseTree(property);
-    }
-  }
-}
-
-function prepareMessagesReport(messages: any) {
-  const idProperties = [
-    "id",
-    "hookId",
-    "testStepId",
-    "testCaseId",
-    "testCaseStartedId",
-    "pickleId",
-    "pickleStepId",
-  ] as const;
-
-  const idCollectionProperties = ["astNodeIds", "stepDefinitionIds"] as const;
-
-  for (const message of messages) {
-    for (const node of traverseTree(message)) {
-      if (hasOwnProperty(node, "duration")) {
-        node.duration = 0;
-      }
-
-      if (hasOwnProperty(node, "timestamp")) {
-        node.timestamp = {
-          seconds: 0,
-          nanos: 0,
-        };
-      }
-
-      if (hasOwnProperty(node, "uri") && typeof node.uri === "string") {
-        node.uri = node.uri.replace(/\\/g, "/");
-      }
-
-      if (hasOwnProperty(node, "meta")) {
-        node.meta = "meta";
-      }
-
-      for (const idProperty of idProperties) {
-        if (hasOwnProperty(node, idProperty)) {
-          node[idProperty] = "id";
-        }
-      }
-
-      for (const idCollectionProperty of idCollectionProperties) {
-        if (hasOwnProperty(node, idCollectionProperty)) {
-          node[idCollectionProperty] = (node[idCollectionProperty] as any).map(
-            () => "id"
-          );
-        }
-      }
-    }
-  }
-
-  return messages;
-}
-
-function stringToNdJson(content: string) {
-  return content
-    .toString()
-    .trim()
-    .split("\n")
-    .map((line: any) => JSON.parse(line));
-}
-
-function ndJsonToString(ndjson: any) {
-  return ndjson.map((o: any) => JSON.stringify(o)).join("\n") + "\n";
-}
-
 async function readMessagesReport(
   cwd: string,
   options: { normalize: boolean } = { normalize: true }

features/support/ICustomWorld.ts

@@ -19,4 +19,6 @@ export default interface ICustomWorld {
   runCypress(options?: ExtraOptions): Promise<void>;
 
   runDiagnostics(options?: ExtraOptions): Promise<void>;
+
+  runMergeMessages(options?: ExtraOptions): Promise<void>;
 }