feat: ensure all tests from the expect JSON file are present in the t…

…est results (#33)

* add a test

* removing matched expected tests

* print all expected but unmatched tests

* remove failing test

* update the README

.github/workflows/ci.yml

@@ -41,6 +41,10 @@ jobs:
       - name: --expect example 2 💎
         run: npm run test:expect:2
 
+      # for https://github.com/bahmutov/cypress-expect/issues/32
+      - name: --expect example 3 💎
+        run: npm run test:expect:3
+
       - name: Semantic Release 🚀
         uses: cycjimmy/semantic-release-action@v2
         env:

README.md

@@ -67,6 +67,14 @@ $ npx cypress-expect run --expect ./cypress/failing/expected.json
 
 **Tip 2:** you do not have to remember the precise name of each test status, the JSON file can use synonyms, like `passed`, `pass`, `passing`, `pass`.
 
+**Tip 3:** if there are more tests listed in the expected JSON file, the process will error with the list of tests not found
+
+```text
+cypress-expect: expected the find the following tests
+* login tests / first test
+* authentication test
+```
+
 ## Debugging
 
 Run this script with environment variable `DEBUG=cypress-expect` to see verbose logs

cypress/all-matched/expected.json

@@ -0,0 +1,7 @@
+{
+  "should have all expected tests": {
+    "passes": "passing",
+    "is pending": "pending"
+  },
+  "works": "pending"
+}

cypress/all-matched/spec.js

@@ -0,0 +1,11 @@
+/// <reference types="cypress" />
+
+describe('should have all expected tests', () => {
+  it('passes', () => {
+    cy.wait(100)
+  })
+
+  it('is pending')
+})
+
+it('works')

index.js

@@ -9,6 +9,7 @@ const debug = require('debug')('cypress-expect')
 const arg = require('arg')
 const fs = require('fs')
 const R = require('ramda')
+const allPaths = require('@bahmutov/all-paths')
 
 const REPO_URL = 'https://github.com/bahmutov/cypress-expect'
 const CLI_HELP_URL = 'https://github.com/bahmutov/cypress-expect#options'
@@ -167,6 +168,34 @@ const parseArguments = async () => {
   return await cypress.cli.parseRunArguments(cliArgs)
 }
 
+// NOTE: modifies the object in place
+const removeEmptyLeaves = (json) => {
+  if (typeof json === 'string') {
+    return
+  }
+  Object.keys(json).forEach((key) => {
+    removeEmptyLeaves(json[key])
+    if (R.isEmpty(json[key])) {
+      delete json[key]
+    }
+  })
+
+  return json
+}
+
+const collectPaths = (json) => {
+  const list = allPaths(json)
+    // leave only paths that lead to the test status
+    // which is a string, effectively filtering out
+    // all outer suite names
+    .filter((path) => {
+      return typeof R.path(path, json) === 'string'
+    })
+  return list
+}
+
+const testTitle = (parts) => parts.join(' / ')
+
 parseArguments()
   .then((options) => {
     debug('parsed CLI options %o', options)
@@ -203,11 +232,12 @@ parseArguments()
     }
 
     if (isExpectSpecified) {
-      const expectedTestStatuses = getExpectedTestStatuses(args['--expect'])
+      // a single object with expected test results
+      let expectedTestStatuses = getExpectedTestStatuses(args['--expect'])
       debug('expected test statuses %o', expectedTestStatuses)
 
       debug('test runs %o', runResults.runs)
-      // collect every test result
+      // collect every test result reported by Cypress
       const tests = []
       runResults.runs.forEach((runResult) => {
         runResult.tests.forEach((testResult) => {
@@ -241,6 +271,12 @@ parseArguments()
             )
           }
         } else {
+          // found the expected test record in the JSON file
+          // let's remove it - by the end of the matching the "expected"
+          // object will only have expected test results that were NOT
+          // present in the test results
+          expectedTestStatuses = R.dissocPath(test.title, expectedTestStatuses)
+
           const normalized = normalizeTestState(expectedTestStatus)
           debug(
             'test "%s" should status "%s"',
@@ -271,6 +307,17 @@ parseArguments()
         process.exit(1)
       }
 
+      const removedEmpty = removeEmptyLeaves(expectedTestStatuses)
+      if (!R.isEmpty(removedEmpty)) {
+        console.error('cypress-expect: expected the find the following tests')
+        const titles = collectPaths(removedEmpty)
+          .map(testTitle)
+          .map((s) => '* ' + s)
+        console.error(titles.join('\n'))
+        console.error()
+        process.exit(1)
+      }
+
       // nothing else to do
       return
     }

package.json

@@ -12,7 +12,8 @@
     "test": "node . run --passing 3",
     "semantic-release": "semantic-release",
     "test:expect:1": "node . run --expect ./cypress/integration/expected.json",
-    "test:expect:2": "node . run --expect ./cypress/failing/expected.json --config integrationFolder=cypress/failing"
+    "test:expect:2": "node . run --expect ./cypress/failing/expected.json --config integrationFolder=cypress/failing",
+    "test:expect:3": "node . run --expect ./cypress/all-matched/expected.json --config integrationFolder=cypress/all-matched"
   },
   "repository": {
     "type": "git",
@@ -35,6 +36,7 @@
     "semantic-release": "^17.1.1"
   },
   "dependencies": {
+    "@bahmutov/all-paths": "1.0.2",
     "arg": "5.0.0",
     "debug": "4.3.1",
     "ramda": "0.27.1"