Merge pull request #20 from yachr/GH-18-Roll-up-steps-to-scenarios-to…

…-features

Gh 18 roll up steps to scenarios to features

.gitignore

@@ -12,3 +12,7 @@ reports/**/*.html
 .nyc_output/
 .coveralls.yml
 *.tgz
+
+e2e/reportOutput
+# Transpiled E2E
+e2e/**/*.js

.travis.yml

@@ -6,7 +6,7 @@ env:
 node_js:
   - '8.12'
 script:
-  - npm ci
+  - npm run ci
 
 branches:
   except:

.vscode/settings.json

@@ -1,9 +1,17 @@
 {
     "workbench.colorCustomizations": {
-        "titleBar.activeBackground": "#38224c",
-        "titleBar.inactiveBackground": "#38224c99",
-        "titleBar.activeForeground": "#e7e7e7",
-        "titleBar.inactiveForeground": "#e7e7e799"
+      "activityBar.background": "#52326f",
+      "activityBar.foreground": "#e7e7e7",
+      "activityBar.inactiveForeground": "#e7e7e799",
+      "activityBarBadge.background": "#000000",
+      "activityBarBadge.foreground": "#e7e7e7",
+      "titleBar.activeBackground": "#38224c",
+      "titleBar.inactiveBackground": "#38224c99",
+      "titleBar.activeForeground": "#e7e7e7",
+      "titleBar.inactiveForeground": "#e7e7e799",
+      "statusBar.background": "#38224c",
+      "statusBarItem.hoverBackground": "#52326f",
+      "statusBar.foreground": "#e7e7e7"
     },
     "python.linting.pylintEnabled": true,
     "cucumberautocomplete.steps": [
@@ -14,5 +22,6 @@
   "cucumberautocomplete.smartSnippets": true,
   "cucumberautocomplete.stepsInvariants": true,
   "cucumberautocomplete.onTypeFormat": true,
-  "cucumberautocomplete.gherkinDefinitionPart": "(Given|When|Then)\\("
+  "cucumberautocomplete.gherkinDefinitionPart": "(Given|When|Then)\\(",
+  "peacock.color": "#38224c"
 }

README.md

@@ -1,6 +1,6 @@
 # YACHR
-[![Build Status](https://travis-ci.org/yachr/yachr.svg?branch=develop)](https://travis-ci.org/yachr/yachr/branches)
-[![Coverage Status](https://coveralls.io/repos/github/yachr/yachr/badge.svg?branch=GH-13-CodeCoverage)](https://coveralls.io/github/yachr/yachr?branch=GH-13-CodeCoverage)
+[![Build Status](https://travis-ci.org/yachr/yachr.svg?branch=GH-18-Roll-up-steps-to-scenarios-to-features)](https://travis-ci.org/yachr/yachr/branches)
+[![Coverage Status](https://coveralls.io/repos/github/yachr/yachr/badge.svg?branch=GH-18-Roll-up-steps-to-scenarios-to-features)](https://coveralls.io/github/yachr/yachr?branch=GH-18-Roll-up-steps-to-scenarios-to-features)
 
 Yet another cucumber html reporter is a simple html reporter that runs off the standard json file produced by cucumberjs.
 
@@ -91,6 +91,22 @@ From the root:
 
 Should produced `dist/samples/report.html`
 
+## Testing out changes to the html template
+When making changes to the template its good to see those changes applied as you go.
+
+One option to do this if you're using the templating system, is to get the unit tests to run as you make changes.
+
+As a starting point, the 'should generate a report' test in the reporter.spec.ts will generate a basic looking report.
+
+This can be run on its own by updating the test's `it` to use the only function:
+
+```
+it.only('should generate a report', () => {
+```
+
+Don't commit this line though!
+
+One last step is to comment out the code that cleans up the test and removes the generated html. This should be the last line of the test, and has a helpful comment to point it out for you.
 # CI
 yachr is monitored by Travis-ci. when a change is detected Travis-ci will pull the repo and execute `npm run ci`. Travis will run `ci` before accepting a pull request.
 

e2e/features/abilities/user/view-report-summary.feature

@@ -1,14 +1,52 @@
-  Ability: View report summary
+Ability: View report summary
 
   As a user
   I would like to see the summary of features and scenarios in my project
   So that I can gauge my project's health and progress.
 
-  Scenario: Feature summary
+  The status of a Scenario behaves like a hierarchy that rolls up.
+  The scenario status will be the 'worst' status of its child steps as follows:
+  ambiguous, failed, undefined, pending, passed
+  Although a step can be skipped, a scneario cannot.
+
+  Ambiguous is the worst because it is similar to a compile erorr. There are
+  two or more implementations that match one step, and the test simply can't be run.
+
+  Failed is next because a step has been implemented, and failed, which is unexpected.
+
+  Undefined is then next, because no implementation has been put together.
+
+  Pending is where the implementation exists, but returns the string pending.
+
+  Finally, if all steps pass, then the scenario passes.
+
+  Scenario: All passing
     Given a passing scenario
       | Feature     | Scenario     | Step   | Step Status |
       | Feature One | Scenario One | Step 1 | passed      |
       | Feature One | Scenario One | Step 2 | passed      |
       | Feature One | Scenario One | Step 3 | passed      |
     When I run yachr against it
     Then a summary showing one passing feature and one passing scenario
+
+  Scenario: Handle mixed states
+    Given the following scenarios
+      | Feature     | Scenario     | Step   | Step Status |
+      | Feature One | Scenario One | Step 1 | passed      |
+      | Feature One | Scenario One | Step 2 | ambiguous   |
+
+      | Feature Two | Scenario One | Step 1 | passed |
+      | Feature Two | Scenario One | Step 2 | failed |
+
+      | Feature Three | Scenario One | Step 1 | passed  |
+      | Feature Three | Scenario One | Step 2 | pending |
+
+      | Feature Four | Scenario One | Step 1 | undefined |
+
+    When I run yachr against it
+    Then I will see the following in the summary
+      | Feature       | Status    |
+      | Feature One   | ambiguous |
+      | Feature Two   | failed    |
+      | Feature three | pending   |
+      | Feature four  | undefined |

e2e/resources/allStatuses.json

@@ -0,0 +1,72 @@
+[
+  {
+    "description": "Feature One",
+    "keyword": "Ability",
+    "name": "View report summary",
+    "id": "view-report-summary",
+    "tags": [],
+    "elements": [
+      {
+        "id": "view-report-summary;feature-summary",
+        "keyword": "Scenario",
+        "line": 7,
+        "name": "Feature summary",
+        "tags": [],
+        "type": "scenario",
+        "steps": [
+          {
+            "arguments": [],
+            "keyword": "Given ",
+            "result": {
+              "status": "passed",
+              "duration": 1000000
+            }
+          },
+          {
+            "arguments": [],
+            "keyword": "Then",
+            "result": {
+              "status": "ambigious",
+              "duration": 1000000
+            }
+          }
+        ]
+      }
+    ]
+  },
+  {
+    "description": "Feature Two",
+    "keyword": "Ability",
+    "name": "View report summary",
+    "id": "view-report-summary",
+    "tags": [],
+    "elements": [
+      {
+        "id": "view-report-summary;feature-summary",
+        "keyword": "Scenario",
+        "line": 7,
+        "name": "Feature summary",
+        "tags": [],
+        "type": "scenario",
+        "steps": [
+          {
+            "arguments": [],
+            "keyword": "Given ",
+            "result": {
+              "status": "passed",
+              "duration": 1000000
+            }
+          },
+          {
+            "arguments": [],
+            "keyword": "Then",
+            "result": {
+              "status": "failed",
+              "duration": 1000000
+            }
+          }
+        ]
+      }
+    ]
+  }
+]

e2e/step_definitions/view-report-summary.steps.ts

@@ -8,27 +8,34 @@ import * as passingScenario from '../resources/onePassingScenario.json';
 
 import * as cherrio from 'cheerio';
 const reportLocation = 'e2e/reportOutput/report.html';
+const jsonOutput = 'e2e/resources/onePassingScenario.json';
 
-let jsonOutput;
 let reporter: Reporter;
 let reportOptions: IReportOptions;
 
-Given('a passing scenario', function (table) {
-  reportOptions = <IReportOptions>{
-    jsonFile: 'e2e/resources/onePassingScenario.json',
+Given('a passing scenario', table => {
+  reportOptions = <IReportOptions> {
+    jsonFile: jsonOutput,
     output: reportLocation,
   };
 });
 
-When('I run yachr against it', function () {
+Given('the following scenarios', table =>
+  'pending');
+
+When('I run yachr against it', () => {
   reporter = new Reporter();
   reporter.generate(reportOptions);
 });
 
-Then('a summary showing one passing feature and one passing scenario', async function () {
+Then('a summary showing one passing feature and one passing scenario', async () => {
   const $ = cherrio.load(fs.readFileSync(reportLocation, 'utf8'));
   const pageText = $('html').text();
 
-  const passedFeatures = /Features([.\n\s]*)1/;
+  // ([.\n\s]*)1/ Match all spaces and new line chars
+  const passedFeatures = /Ability: View report summary([.\n\s]*)done1/;
   expect(passedFeatures.test(pageText)).eql(true);
 });
+
+Then('I will see the following in the summary', () =>
+  'pending');

src/models/aggregator/featureSuiteSummary.ts

@@ -9,24 +9,28 @@ export class FeatureSuiteSummary {
 
   public passingFeatures: FeatureSummary[] = [];
   public failingFeatures: FeatureSummary[] = [];
+  public ambiguousFeatures: FeatureSummary[] = [];
+  public pendingFeatures: FeatureSummary[] = [];
   public undefinedFeatures: FeatureSummary[] = [];
-  public partialFeatures: FeatureSummary[] = [];
 
-  /** The total number of features that have passed */
-  public get passed(): number { return this.passingFeatures.length; }
+  /** The total number of Scenarios that are ambiguously defined for the Feature */
+  public get ambiguous(): number { return this.ambiguousFeatures.length; }
 
   /** The total number of features that have failed */
   public get failed(): number { return this.failingFeatures.length; }
 
   /** The total number of features that are not implemented (and thus marked as undefined) */
   public get undefined(): number { return this.undefinedFeatures.length; }
 
-  /** Keeps track of partially passing features */
-  public get partial(): number { return this.partialFeatures.length; }
+  /** The total number of features that have passed */
+  public get pending(): number { return this.pendingFeatures.length; }
+
+  /** The total number of features that have passed */
+  public get passed(): number { return this.passingFeatures.length; }
 
   /** All features in this group  */
   get total(): number {
-    return this.passed + this.failed + this.undefined + this.partial;
+    return this.ambiguous + this.failed + this.undefined + this.pending + this.passed;
   }
 
   /** Whether the Suite or Feature has failed due to a failed Feature or Scenario */
@@ -40,10 +44,13 @@ export class FeatureSuiteSummary {
 
   /** Updates the aggregated summary using information gathered in the Element Summary */
   public aggregateFeature(feature: FeatureSummary): void {
-    if (feature.isFailed) { this.failingFeatures.push(feature); }
+
+    if (feature.hasAmbiguous) { this.ambiguousFeatures.push(feature); }
+    else if (feature.hasFailed) { this.failingFeatures.push(feature); }
+    else if (feature.hasUndefined) { this.undefinedFeatures.push(feature); }
+    else if (feature.hasPending) { this.pendingFeatures.push(feature); }
     else if (feature.isPassed) { this.passingFeatures.push(feature); }
-    else if (feature.isUndefined) { this.undefinedFeatures.push(feature); }
-    else { this.partialFeatures.push(feature); }
+    else { throw new Error('Scenario cannot be aggregated. Please raise a GitHub issue with the Yachr Team'); }
   }
 
 }

src/models/aggregator/featureSummary.ts

@@ -8,44 +8,76 @@ import { ScenarioSummary } from './scenarioSummary';
 export class FeatureSummary {
 
   public featureName: string = '';
+  public featureDescription: string = '';
+  public featureKeyword: string = ''; // Ability / Feature
 
   public passingScenarios: ScenarioSummary[] = [];
   public failingScenarios: ScenarioSummary[] = [];
+  public ambiguousScenarios: ScenarioSummary[] = [];
+  public pendingScenarios: ScenarioSummary[] = [];
   public undefinedScenarios: ScenarioSummary[] = [];
-  public partialScenarios: ScenarioSummary[] = [];
 
-  /** The total number of Scenarios that have passed in this Feature */
-  public get passed(): number { return this.passingScenarios.length; }
+  /** The total number of Scenarios that are ambiguously defined for the Feature */
+  public get ambiguous(): number { return this.ambiguousScenarios.length; }
 
   /** The total number of Scenarios that have failed in this Feature */
   public get failed(): number { return this.failingScenarios.length; }
 
-  /** The total number of Scenarios that are not implemented (and thus marked as undefined) */
+  /** The total number of Scenarios that are have no step definition (and thus marked as undefined) */
   public get undefined(): number { return this.undefinedScenarios.length; }
 
-  /** Keeps track of partially passing statuses */
-  public get partial(): number { return this.partialScenarios.length; }
+  /** The total number of Scenarios that are not implemented (and thus marked as pending) */
+  public get pending(): number { return this.pendingScenarios.length; }
+
+  /** The total number of Scenarios that have passed in this Feature */
+  public get passed(): number { return this.passingScenarios.length; }
 
   /** All Scenarios in this Feature  */
   get total(): number {
-    return this.passed + this.failed + this.undefined + this.partial;
+    return this.ambiguous + this.failed + this.undefined + this.pending + this.passed;
   }
 
-  /** Whether the Feature has failed due to a failed Scenario */
-  get isFailed(): boolean { return this.failed > 0; }
+  /** Whether the Feature has at least one ambiguous Scenario */
+  get hasAmbiguous(): boolean { return this.ambiguous > 0; }
+
+  /** Whether the Feature has at least one failed Scenario */
+  get hasFailed(): boolean { return this.failed > 0; }
+
+  /** Whether the Feature has at least one undefined Scenario */
+  get hasUndefined(): boolean { return this.undefined > 0; }
+
+  /** Whether the Feature has at least one pending Scenario */
+  get hasPending(): boolean {
+    return this.pending > 0 ||
+    this.hasNoScenarios;
+  }
+
+  get hasNoScenarios(): boolean { return this.total == 0; }
 
   /** Whether the Feature has passed due to all Scenarios passing */
   get isPassed(): boolean { return this.passed === this.total; }
 
-  /** Whether the entire Feature is entirely undefined */
-  get isUndefined(): boolean { return this.undefined === this.total; }
-
   /** Updates the Feature summary using information gathered in the Scenario Summary */
   public aggregateScenario(scenario: ScenarioSummary): void {
-    if (scenario.isFailed) { this.failingScenarios.push(scenario); }
+    // As per view-report-summary.feature
+    // The status of a Scenario behaves like a hierarchy that rolls up.
+    // The scenario status will be the 'worst' status of its child steps as follows:
+    // ambiguous, failed, undefined, pending, passed
+    // Although a step can be skipped, a scenario cannot.
+
+    // - Ambiguous is the worst because it is similar to a compile erorr. There are
+    //   two or more implementations that match one step, and the test simply can't be run.
+    // - Failed is next because a step has been implemented, and failed, which is unexpected.
+    // - Undefined is then next, because no implementation has been put together.
+    // - Pending is where the implementation exists, but returns the string pending.
+    // - Finally, if all steps pass, then the scenario passes.
+
+    if (scenario.hasAmbiguous) { this.ambiguousScenarios.push(scenario); }
+    else if (scenario.hasFailed) { this.failingScenarios.push(scenario); }
+    else if (scenario.hasUndefined) { this.undefinedScenarios.push(scenario); }
+    else if (scenario.hasPending) { this.pendingScenarios.push(scenario); }
     else if (scenario.isPassed) { this.passingScenarios.push(scenario); }
-    else if (scenario.isUndefined) { this.undefinedScenarios.push(scenario); }
-    else { this.partialScenarios.push(scenario); }
+    else { throw new Error('Scenario cannot be aggregated. Please raise a GitHub issue with the Yachr Team'); }
   }
 
 }