test(system-tests): support docker-based tests against built binary

__snapshots__/bump-spec.js

@@ -3,16 +3,6 @@ exports['list of all projects'] = [
     "repo": "cypress-io/cypress-test-module-api",
     "provider": "circle",
     "platform": "linux"
-  },
-  {
-    "repo": "cypress-io/cypress-test-node-versions",
-    "provider": "circle",
-    "platform": "linux"
-  },
-  {
-    "repo": "cypress-io/cypress-test-ci-environments",
-    "provider": "circle",
-    "platform": "linux"
   }
 ]
 
@@ -21,15 +11,5 @@ exports['should have just circle and linux projects'] = [
     "repo": "cypress-io/cypress-test-module-api",
     "provider": "circle",
     "platform": "linux"
-  },
-  {
-    "repo": "cypress-io/cypress-test-node-versions",
-    "provider": "circle",
-    "platform": "linux"
-  },
-  {
-    "repo": "cypress-io/cypress-test-ci-environments",
-    "provider": "circle",
-    "platform": "linux"
   }
 ]

circle.yml

@@ -11,7 +11,7 @@ defaults: &defaults
       type: boolean
       default: false
   executor: <<parameters.executor>>
-  environment:
+  environment: &defaultsEnvironment
     ## set specific timezone
     TZ: "/usr/share/zoneinfo/America/New_York"
 
@@ -169,7 +169,7 @@ commands:
           command: node scripts/circle-cache.js --action cacheKey > circle_cache_key
       - restore_cache:
           name: Restore cache state, to check for known modules cache existence
-          key: v{{ .Environment.CACHE_VERSION }}-{{ arch }}-test2-node-modules-cache-{{ checksum "circle_cache_key" }}
+          key: v{{ .Environment.CACHE_VERSION }}-node-modules-cache-{{ checksum "circle_cache_key" }}
       - run:
           name: Move node_modules back from /tmp
           command: |
@@ -193,8 +193,8 @@ commands:
       - restore_cache:
           name: Restore system tests node_modules cache
           keys:
-            - v{{ .Environment.CACHE_VERSION }}-{{ arch }}-system-tests-projects-node-modules-cache-{{ checksum "system_tests_cache_key" }}
-            - v{{ .Environment.CACHE_VERSION }}-{{ arch }}-system-tests-projects-node-modules-cache-
+            - v{{ .Environment.CACHE_VERSION }}-system-tests-projects-node-modules-cache-{{ checksum "system_tests_cache_key" }}
+            - v{{ .Environment.CACHE_VERSION }}-system-tests-projects-node-modules-cache-
 
   update_cached_system_tests_deps:
     description: 'Update the cached node_modules for projects in "system-tests/projects/**"'
@@ -205,36 +205,36 @@ commands:
       - restore_cache:
           name: Restore cache state, to check for known modules cache existence
           keys:
-            - v{{ .Environment.CACHE_VERSION }}-{{ arch }}-system-tests-projects-node-modules-cache-state-{{ checksum "system_tests_cache_key" }}
+            - v{{ .Environment.CACHE_VERSION }}-system-tests-projects-node-modules-cache-state-{{ checksum "system_tests_cache_key" }}
       - run:
           name: Send root honeycomb event for this CI build
           command: cd system-tests/scripts && node ./send-root-honecomb-event.js
       - run:
           name: Bail if specific cache exists
           command: |
-            if [[ -f "system_tests_node_modules_installed" ]]; then
+            if [[ -f "/tmp/system_tests_node_modules_installed" ]]; then
               echo "No updates to system tests node modules, exiting"
               circleci-agent step halt
             fi
       - restore_cache:
           name: Restore system tests node_modules cache
           keys:
-            - v{{ .Environment.CACHE_VERSION }}-{{ arch }}-system-tests-projects-node-modules-cache-{{ checksum "system_tests_cache_key" }}
-            - v{{ .Environment.CACHE_VERSION }}-{{ arch }}-system-tests-projects-node-modules-cache-
+            - v{{ .Environment.CACHE_VERSION }}-system-tests-projects-node-modules-cache-{{ checksum "system_tests_cache_key" }}
+            - v{{ .Environment.CACHE_VERSION }}-system-tests-projects-node-modules-cache-
       - run:
           name: Update system-tests node_modules cache
           command: yarn workspace @tooling/system-tests projects:yarn:install
       - save_cache:
           name: Save system tests node_modules cache
-          key: v{{ .Environment.CACHE_VERSION }}-{{ arch }}-system-tests-projects-node-modules-cache-{{ checksum "system_tests_cache_key" }}
+          key: v{{ .Environment.CACHE_VERSION }}-system-tests-projects-node-modules-cache-{{ checksum "system_tests_cache_key" }}
           paths:
             - ~/.cache/cy-system-tests-node-modules
-      - run: touch system_tests_node_modules_installed
+      - run: touch /tmp/system_tests_node_modules_installed
       - save_cache:
           name: Save system tests node_modules cache state key
-          key: v{{ .Environment.CACHE_VERSION }}-{{ arch }}-system-tests-projects-node-modules-cache-state-{{ checksum "system_tests_cache_key" }}
+          key: v{{ .Environment.CACHE_VERSION }}-system-tests-projects-node-modules-cache-state-{{ checksum "system_tests_cache_key" }}
           paths:
-            - system_tests_node_modules_installed
+            - /tmp/system_tests_node_modules_installed
 
   caching-dependency-installer:
     description: 'Installs & caches the dependencies based on yarn lock & package json dependencies'
@@ -249,19 +249,19 @@ commands:
           command: node scripts/circle-cache.js --action cacheKey > circle_cache_key
       - restore_cache:
           name: Restore cache state, to check for known modules cache existence
-          key: v{{ .Environment.CACHE_VERSION }}-{{ arch }}-test2-node-modules-cache-state-{{ checksum "circle_cache_key" }}
+          key: v{{ .Environment.CACHE_VERSION }}-node-modules-cache-state-{{ checksum "circle_cache_key" }}
       - run:
           name: Bail if cache exists
           command: |
-            if [[ -f "node_modules_installed" ]]; then
+            if [[ -f "/tmp/node_modules_installed" ]]; then
               echo "Node modules already cached for dependencies, exiting"
               circleci-agent step halt
             fi
       - run: date +%Y-%U > cache_date
       - restore_cache:
           name: Restore weekly yarn cache
           keys:
-            - v{{ .Environment.CACHE_VERSION }}-{{ arch }}-test2-deps-root-weekly-{{ checksum "cache_date" }}
+            - v{{ .Environment.CACHE_VERSION }}-deps-root-weekly-{{ checksum "cache_date" }}
       - run:
           name: Install Node Modules
           command: |
@@ -274,7 +274,7 @@ commands:
           steps:
             - save_cache:
                 name: Saving node modules for root, cli, and all globbed workspace packages
-                key: v{{ .Environment.CACHE_VERSION }}-{{ arch }}-test2-node-modules-cache-{{ checksum "circle_cache_key" }}
+                key: v{{ .Environment.CACHE_VERSION }}-node-modules-cache-{{ checksum "circle_cache_key" }}
                 paths:
                   - node_modules
                   - cli/node_modules
@@ -285,18 +285,18 @@ commands:
           steps:
             - save_cache:
                 name: Saving node modules for root, cli, and all globbed workspace packages
-                key: v{{ .Environment.CACHE_VERSION }}-{{ arch }}-test2-node-modules-cache-{{ checksum "circle_cache_key" }}
+                key: v{{ .Environment.CACHE_VERSION }}-node-modules-cache-{{ checksum "circle_cache_key" }}
                 paths:
                   - /tmp/node_modules_cache
-      - run: touch node_modules_installed
+      - run: touch /tmp/node_modules_installed
       - save_cache:
           name: Saving node-modules cache state key
-          key: v{{ .Environment.CACHE_VERSION }}-{{ arch }}-test2-node-modules-cache-state-{{ checksum "circle_cache_key" }}
+          key: v{{ .Environment.CACHE_VERSION }}-node-modules-cache-state-{{ checksum "circle_cache_key" }}
           paths:
-            - node_modules_installed
+            - /tmp/node_modules_installed
       - save_cache:
           name: Save weekly yarn cache
-          key: v{{ .Environment.CACHE_VERSION }}-{{ arch }}-test2-deps-root-weekly-{{ checksum "cache_date" }}
+          key: v{{ .Environment.CACHE_VERSION }}-deps-root-weekly-{{ checksum "cache_date" }}
           paths:
             - ~/.yarn
 
@@ -519,6 +519,24 @@ commands:
           path: /tmp/artifacts
       - store-npm-logs
 
+  run-binary-system-tests:
+    steps:
+      - restore_cached_workspace
+      - restore_cached_system_tests_deps
+      - run:
+          name: Run system tests
+          command: |
+            ALL_SPECS=`circleci tests glob "$HOME/cypress/system-tests/test-binary/*spec*"`
+            SPECS=`echo $ALL_SPECS | xargs -n 1 | circleci tests split --split-by=timings`
+            echo SPECS=$SPECS
+            yarn workspace @tooling/system-tests test:ci $SPECS
+      - verify-mocha-results
+      - store_test_results:
+          path: /tmp/cypress
+      - store_artifacts:
+          path: /tmp/artifacts
+      - store-npm-logs
+
   store-npm-logs:
     description: Saves any NPM debug logs as artifacts in case there is a problem
     steps:
@@ -1162,6 +1180,19 @@ jobs:
       - restore_cached_workspace
       - update_cached_system_tests_deps
 
+  binary-system-tests:
+    parallelism: 2
+    working_directory: ~/cypress
+    environment:
+      <<: *defaultsEnvironment
+    machine:
+      # using `machine` gives us a Linux VM that can run Docker
+      image: ubuntu-2004:202111-02
+      docker_layer_caching: true
+    resource_class: medium
+    steps:
+      - run-binary-system-tests
+
   system-tests-chrome:
     <<: *defaults
     resource_class: medium
@@ -2279,6 +2310,10 @@ linux-workflow: &linux-workflow
         name: "test binary as a root user"
         requires:
           - create-build-artifacts
+    - binary-system-tests:
+        requires:
+          - create-build-artifacts
+          - system-tests-node-modules-install
 
 mac-workflow: &mac-workflow
   jobs:

guides/release-process.md

@@ -197,10 +197,8 @@ In the following instructions, "X.Y.Z" is used to denote the [next version of Cy
     - [cypress-example-todomvc-redux](https://github.com/cypress-io/cypress-example-todomvc-redux/issues/1)
     - [cypress-example-realworld](https://github.com/cypress-io/cypress-example-realworld/issues/2)
     - [cypress-example-recipes](https://github.com/cypress-io/cypress-example-recipes/issues/225)
-    - [cypress-example-api-testing](https://github.com/cypress-io/cypress-example-api-testing/issues/15)
     - [angular-pizza-creator](https://github.com/cypress-io/angular-pizza-creator/issues/5)
     - [cypress-fiddle](https://github.com/cypress-io/cypress-fiddle/issues/5)
-    - [cypress-example-piechopper](https://github.com/cypress-io/cypress-example-piechopper/issues/75)
     - [cypress-documentation](https://github.com/cypress-io/cypress-documentation/issues/1313)
     - [cypress-example-docker-compose](https://github.com/cypress-io/cypress-example-docker-compose) - Doesn't have a Renovate issue, but will auto-create and auto-merge non-major Cypress updates as long as the tests pass.
 
@@ -209,11 +207,6 @@ In the following instructions, "X.Y.Z" is used to denote the [next version of Cy
     **Test Repos**
 
     - [cypress-test-tiny](https://github.com/cypress-io/cypress-test-tiny)
-    - [cypress-test-nested-projects](https://github.com/cypress-io/cypress-test-nested-projects)
-    - [cypress-test-example-repos](https://github.com/cypress-io/cypress-test-example-repos)
-    - [cypress-test-node-versions](https://github.com/cypress-io/cypress-test-node-versions)
-    - [cypress-test-module-api](https://github.com/cypress-io/cypress-test-module-api)
-    - [cypress-test-ci-environments](https://github.com/cypress-io/cypress-test-ci-environments)
 
     **Example Repos**
 
@@ -222,8 +215,6 @@ In the following instructions, "X.Y.Z" is used to denote the [next version of Cy
     - [cypress-example-realworld](https://github.com/cypress-io/cypress-example-realworld)
     - [cypress-example-recipes](https://github.com/cypress-io/cypress-example-recipes)
     - [cypress-example-docker-compose](https://github.com/cypress-io/cypress-example-docker-compose)
-    - [cypress-example-api-testing](https://github.com/cypress-io/cypress-example-api-testing)
-    - [cypress-example-piechopper](https://github.com/cypress-io/cypress-example-piechopper)
     - [cypress-documentation](https://github.com/cypress-io/cypress-documentation)
 
 Take a break, you deserve it! :sunglasses:

scripts/binary/bump.js

@@ -17,8 +17,6 @@ const _PROVIDERS = {
     main: 'cypress-io/cypress',
     linux: [
       'cypress-io/cypress-test-module-api',
-      'cypress-io/cypress-test-node-versions',
-      'cypress-io/cypress-test-ci-environments',
     ],
   },
 }

system-tests/README.md

@@ -49,7 +49,7 @@ describe('my new project', () => {
     systemTests.setup()
 
     systemTests.it('fails as expected', {
-        project: Fixtures.projectPath('my-new-project'),
+        project: 'my-new-project',
         snapshot: true,
         spec: '*',
         expectedExitCode: 2
@@ -61,6 +61,36 @@ From here, you could run this test with `yarn test my-new-project`.
 
 There are many more options available for `systemTests.it` and `systemTests.setup`. You can massage the stdout, do pre-run tasks, set up HTTP/S servers, and more. Explore the typedocs in [`./lib/system-tests`](./lib/system-tests) for more information.
 
+These tests run in the `system-tests-*` CI jobs.
+
+### Developing Docker-based tests against built binary
+
+Specs in the [`./test`](./test) directory are run against an unbuilt Cypress App. They don't test `cypress` NPM package installation or other prod app behavior. This is done so that they can run as fast as possible in CI, without waiting for a full build of the Cypress App.
+
+Specs in [`./test-binary`](./test-binary) are run against the *built Cypress App*. They also run inside of their own Docker containers to give a blank slate environment for Cypress to run in. Before each test, the prod CLI is `npm install`ed along with the built Cypress `.zip`, and real `cypress run` commands are used to run the tests. There should be no functional difference between running a project in these tests and running real prod Cypress inside of Docker in CI.
+
+The purpose of these tests is to test things that we normally can't inside of regular `system-tests`, such as testing Cypress with different Node versions, with/without Xvfb, or inside of different operating system versions.
+
+An example of using `dockerImage` and `withBinary` to write a binary system test:
+
+```ts
+// ./test-binary/node-versions.spec.ts
+import systemTests from '../lib/system-tests'
+import Fixtures from '../lib/fixtures'
+
+describe('node versions', () => {
+    systemTests.it('runs in node 12', {
+        dockerImage: 'cypress:node/12',
+        project: 'todos',
+        withBinary: true,
+    })
+})
+```
+
+Running `yarn test node-versions` would spin up a local Docker container for `cypress:node/12`, install Cypress from `../cypress.zip` and `../cli/build`, and then call the regular `cypress run` command within the container. Other options for `systemTests.it` such as `onRun` and `expectedExitCode` still function normally.
+
+These tests run in the `binary-system-tests` CI job.
+
 ### Updating Snaphots
 
 Prepend `SNAPSHOT_UPDATE=1` to any test command. See [`snap-shot-it` instructions](https://github.com/bahmutov/snap-shot-it#advanced-use) for more info.