isolated runner changes (#6799)

* isolated runner changes

* more fixes for isolated runner, get runner.spec passing

* fix prev commit add untracked files

* add server reporter.spec.js for testing mocha output

* fix has-binary yarn.lock

* fix spec expected error message

* cleanup snapshot utility - use chai assert fn, add test to runner.spec, pin dep

* remove test-retries related tests

* small cleanup diff utility

* move runner.spec to packages/runner

* fix requiring missing file

* fix circle.yml runner-integration-tests

* update readme

* finish refactor of helpers for isolated runner-integration tests

* restore runner/test/eslintrc

* fix .eslintignore

* temp 04/09/20 [skip ci]

* Update readme to reflect how to start server for cypress tests

* try 2: fix rerun before/after hooks

* temp 04/29/20 [skip ci]

* backport changes from test-retries

* change logic to rerun before hooks after top navigation

* fix windowSize for browser e2e test

* fix windowSize for xvfb chrome in e2e test

* ok fine, just disable screenshots

* fix after merge: decaffed navigation.js

* update server/unit test snapshots

* fix after merge: decaffed aliases.js

* fix usage of cypress --parallel flag in circle.yml

* fix circle.yml integration-tests jobs

* fix decaf related typo

* fix circle.yml separate command for runner-integration-tests

* update runner/integration tests + snapshot after error improvements

* fix runner/integration snapshots for chrome/ff stacktrace differences

* rerun ci

* fix passing --parallel to runner-integration tests

* perf: faster lookup for hooks without runnables

* fix afterAll hook switch logic

* backport to before/after fix

* backport to before/after fix 2

* cleanup from decaf, fix ui/package.json

* update helpers, simplify runner.spec

* fix lint-types errors, flaky spec

* fix noExit passed to e2e test inline options

* cleanup snapshot utility - refactor to use util file

* remove before/after changes

* make cy obj a class instance

* cleanup/unmerge before/after fixes PR...

* more cleanup

* add comment

* fix runner.spec

* cleanup snapshot utility more, cleanup reporter.spec

* minor rename variable

* fix specName in reporterHeader, spec_helper require

* replace reporter specPath usages with spec object from config

* cleanup, fix specs, fix types tests

* fix config spec paths in isolated runner, fix snapshot plugin button

* combine runner.spec.js and runner_spec.js

* fix incorrect merge

* minor minor cleanup

* rename driver/test/cypress to driver/test

* use yarn workspace over lerna for individual package commands

* add error message to driver start

* remove usage of wait-on

* update <reference types/>, import string

* fix driver/readme

* fix readmes after regex replace

* revert wait-on changes

* Revert "revert wait-on changes"

This reverts commit 6de684c.

* update yarn.lock

* fix broken path in spec

* fix broken paths in specs with @packages/driver

* move runner/test/cypress into runner/cypress

* start server in pluginsFile in runner/cypress tests

* fix more broken spec paths

* fix broken paths after runner/cypress folder move

* move type definition loading for driver/cypress into dedicated file

* move internal-types to "types" folder, fix driver/index.d.ts

* fix type-check in packages/runner. not exactly sure why

* fix runner type-check by excluding test folder in tsconfig

* bump timeout on e2e/8_error_ui_spec

* update snapshot utility, rename tests in runner/runner.spec, fix README yarn commands

* fix snapshot naming, remove redundant test in reporter_spec

* rename runner snapshot file

* address feedback: move server reporter snapshot specs out

* address feedback: add comment about exposing globals

Co-authored-by: Jennifer Shehane <jennifer@cypress.io>
Co-authored-by: Brian Mann <brian.mann86@gmail.com>

circle.yml

@@ -124,6 +124,25 @@ commands:
           path: /tmp/artifacts
       - store-npm-logs
 
+  run-runner-integration-tests:
+    parameters:
+      browser:
+        description: browser shortname to target
+        type: string
+    steps:
+      - attach_workspace:
+          at: ~/
+      - run:
+          command: |
+            CYPRESS_KONFIG_ENV=production \
+            CYPRESS_RECORD_KEY=$PACKAGES_RECORD_KEY \
+            yarn workspace @packages/runner cypress:run --record --parallel --group runner-integration-<< parameters.browser >> --browser <<parameters.browser>>
+      - store_test_results:
+          path: /tmp/cypress
+      - store_artifacts:
+          path: /tmp/artifacts
+      - store-npm-logs
+
   run-e2e-tests:
     parameters:
       browser:
@@ -653,6 +672,20 @@ jobs:
           browser: firefox
           chunk: "8"
 
+  "runner-integration-tests-chrome":
+    <<: *defaults
+    parallelism: 2
+    steps:
+      - run-runner-integration-tests:
+          browser: chrome
+
+  "runner-integration-tests-firefox":
+    <<: *defaults
+    parallelism: 2
+    steps:
+      - run-runner-integration-tests:
+          browser: firefox
+
   "driver-integration-tests-chrome":
     <<: *defaults
     parallelism: 5
@@ -1485,6 +1518,13 @@ linux-workflow: &linux-workflow
     - driver-integration-tests-firefox:
         requires:
           - build
+    - runner-integration-tests-chrome:
+        requires:
+          - build
+    - runner-integration-tests-firefox:
+        requires:
+          - build
+
     ## TODO: add these back in when flaky tests are fixed
     # - driver-integration-tests-electron:
     #     requires:

packages/driver/cypress/fixtures/isolated-runner.html

@@ -0,0 +1,23 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+  <meta http-equiv="content-type" content="text/html;charset=utf-8" />
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <title></title>
+
+
+  <link rel="stylesheet" href="/isolated-runner/cypress_runner.css">
+</head>
+
+<body>
+  <div id="app"></div>
+  <script type="text/javascript">
+    // set a global so we know the 'top' window
+    window.Cypress = true
+
+  </script>
+  <script type="text/javascript" src="/isolated-runner/cypress_runner.js"></script>
+</body>
+
+</html>

packages/driver/cypress/plugins/server.js

@@ -35,6 +35,9 @@ ports.forEach((port) => {
     })
   })
 
+  // allows us to serve the testrunner into an iframe for testing
+  app.use('/isolated-runner', express.static(path.join(__dirname, '../../../runner/dist')))
+
   app.get('/node_modules/*', (req, res) => {
     return res.sendFile(path.join('node_modules', req.params[0]), {
       root: path.join(__dirname, '../..'),

packages/driver/src/cy/aliases.js

@@ -1,4 +1,3 @@
-/* globals cy */
 const _ = require('lodash')
 
 const $errUtils = require('../cypress/error_utils')
@@ -44,30 +43,30 @@ const validateAlias = (alias) => {
   }
 }
 
-const create = (state) => {
+const create = (cy) => {
   const addAlias = (ctx, aliasObj) => {
     const { alias, subject } = aliasObj
 
-    const aliases = state('aliases') || {}
+    const aliases = cy.state('aliases') || {}
 
     aliases[alias] = aliasObj
-    state('aliases', aliases)
+    cy.state('aliases', aliases)
 
     const remoteSubject = cy.getRemotejQueryInstance(subject)
 
     ctx[alias] = remoteSubject ?? subject
   }
 
   const getNextAlias = () => {
-    const next = state('current').get('next')
+    const next = cy.state('current').get('next')
 
     if (next && (next.get('name') === 'as')) {
       return next.get('args')[0]
     }
   }
 
   const getAlias = (name, cmd, log) => {
-    const aliases = state('aliases') || {}
+    const aliases = cy.state('aliases') || {}
 
     // bail if the name doesnt reference an alias
     if (!aliasRe.test(name)) {
@@ -85,7 +84,7 @@ const create = (state) => {
   }
 
   const getAvailableAliases = () => {
-    const aliases = state('aliases')
+    const aliases = cy.state('aliases')
 
     if (!aliases) {
       return []
@@ -108,7 +107,7 @@ const create = (state) => {
       })
     }
 
-    cmd = cmd ?? ((log && log.get('name')) || state('current').get('name'))
+    cmd = cmd ?? ((log && log.get('name')) || cy.state('current').get('name'))
     displayName = aliasDisplayName(name)
 
     const errPath = availableAliases.length

packages/driver/src/cy/assertions.js

@@ -1,4 +1,3 @@
-/* global cy Cypress */
 const _ = require('lodash')
 const Promise = require('bluebird')
 
@@ -70,14 +69,14 @@ const parseValueActualAndExpected = (value, actual, expected) => {
   return obj
 }
 
-const create = function (state, queue, retryFn) {
+const create = function (Cypress, cy) {
   const getUpcomingAssertions = () => {
-    const index = state('index') + 1
+    const index = cy.state('index') + 1
 
     const assertions = []
 
     // grab the rest of the queue'd commands
-    for (let cmd of queue.slice(index).get()) {
+    for (let cmd of cy.queue.slice(index).get()) {
       // don't break on utilities, just skip over them
       if (cmd.is('utility')) {
         continue
@@ -111,12 +110,12 @@ const create = function (state, queue, retryFn) {
       // them up with existing ones
       cmd.set('assertionIndex', 0)
 
-      if (state('current') != null) {
-        state('current').set('currentAssertionCommand', cmd)
+      if (cy.state('current') != null) {
+        cy.state('current').set('currentAssertionCommand', cmd)
       }
 
       return cmd.get('fn').originalFn.apply(
-        state('ctx'),
+        cy.state('ctx'),
         [subject].concat(cmd.get('args')),
       )
     })
@@ -139,7 +138,7 @@ const create = function (state, queue, retryFn) {
   const verifyUpcomingAssertions = function (subject, options = {}, callbacks = {}) {
     const cmds = getUpcomingAssertions()
 
-    state('upcomingAssertions', cmds)
+    cy.state('upcomingAssertions', cmds)
 
     // we're applying the default assertion in the
     // case where there are no upcoming assertion commands
@@ -234,7 +233,7 @@ const create = function (state, queue, retryFn) {
       }
 
       if (_.isFunction(onRetry)) {
-        return retryFn(onRetry, options)
+        return cy.retry(onRetry, options)
       }
     }
 
@@ -264,7 +263,7 @@ const create = function (state, queue, retryFn) {
         }
 
         // when we do immediately unbind this function
-        state('onBeforeLog', null)
+        cy.state('onBeforeLog', null)
 
         const insertNewLog = (log) => {
           cmd.log(log)
@@ -347,7 +346,7 @@ const create = function (state, queue, retryFn) {
         return insertNewLog(log)
       }
 
-      state('onBeforeLog', setCommandLog)
+      cy.state('onBeforeLog', setCommandLog)
 
       // send verify=true as the last arg
       return assertFn.apply(this, args.concat(true))
@@ -387,16 +386,16 @@ const create = function (state, queue, retryFn) {
     }
 
     const restore = () => {
-      state('upcomingAssertions', [])
+      cy.state('upcomingAssertions', [])
 
       // no matter what we need to
       // restore the assert fn
-      return state('overrideAssert', undefined)
+      return cy.state('overrideAssert', undefined)
     }
 
     // store this in case our test ends early
     // and we reset between tests
-    state('overrideAssert', overrideAssert)
+    cy.state('overrideAssert', overrideAssert)
 
     return Promise
     .reduce(fns, assertions, [subject])
@@ -465,7 +464,7 @@ const create = function (state, queue, retryFn) {
         // if our current command is an assertion type
         isAssertionType(current) ||
         // are we currently verifying assertions?
-        (state('upcomingAssertions') && state('upcomingAssertions').length > 0) ||
+        (cy.state('upcomingAssertions') && cy.state('upcomingAssertions').length > 0) ||
         // did the function have arguments
         functionHadArguments(current)
     }
@@ -510,7 +509,7 @@ const create = function (state, queue, retryFn) {
   const assert = function (...args) {
     // if we've temporarily overriden assertions
     // then just bail early with this function
-    const fn = state('overrideAssert') || assertFn
+    const fn = cy.state('overrideAssert') || assertFn
 
     return fn.apply(this, args)
   }

packages/driver/src/cy/commands/agents.js

@@ -63,7 +63,7 @@ const onInvoke = function (Cypress, obj, args) {
     error: obj.error,
     type: 'parent',
     end: true,
-    snapshot: true,
+    snapshot: !agent._noSnapshot,
     event: true,
     consoleProps () {
       const consoleObj = {}
@@ -218,6 +218,13 @@ module.exports = function (Commands, Cypress, cy, state) {
       return agent
     }
 
+    // disable DOM snapshots during log for this agent
+    agent.snapshot = (bool = true) => {
+      agent._noSnapshot = !bool
+
+      return agent
+    }
+
     agent.as = (alias) => {
       cy.validateAlias(alias)
       cy.addAlias(ctx, {

packages/driver/src/cy/commands/navigation.js

@@ -107,7 +107,7 @@ const specifyFileByRelativePath = (url, log) => {
   })
 }
 
-const aboutBlank = (win) => {
+const aboutBlank = (cy, win) => {
   return new Promise((resolve) => {
     cy.once('window:load', resolve)
 
@@ -198,7 +198,7 @@ const formSubmitted = (Cypress, e) => {
   })
 }
 
-const pageLoading = (bool, state) => {
+const pageLoading = (bool, Cypress, state) => {
   if (state('pageLoading') === bool) {
     return
   }
@@ -215,7 +215,7 @@ const stabilityChanged = (Cypress, state, config, stable) => {
       // if we're currently visiting about blank
       // and becoming unstable for the first time
       // notifiy that we're page loading
-      pageLoading(true, state)
+      pageLoading(true, Cypress, state)
 
       return
     }
@@ -226,7 +226,7 @@ const stabilityChanged = (Cypress, state, config, stable) => {
   }
 
   // let the world know that the app is page:loading
-  pageLoading(!stable, state)
+  pageLoading(!stable, Cypress, state)
 
   // if we aren't becoming unstable
   // then just return now
@@ -1023,7 +1023,7 @@ module.exports = (Commands, Cypress, cy, state, config) => {
           hasVisitedAboutBlank = true
           currentlyVisitingAboutBlank = true
 
-          return aboutBlank(win)
+          return aboutBlank(cy, win)
           .then(() => {
             currentlyVisitingAboutBlank = false