Improve Regression Testing

.dockerignore

@@ -0,0 +1,17 @@
+# OS generated files
+.DS_Store
+.DS_Store?
+._*
+.Spotlight-V100
+.Trashes
+ehthumbs.db
+Thumbs.db
+
+# Map files
+*.map
+
+# Project dependencies
+node_modules
+.nyc_output
+coverage
+/dist

.eslintrc.json

@@ -1,8 +1,10 @@
 {
+    "plugins": ["cypress"],
     "env": {
         "browser": true,
         "es6": true,
-        "jest": true
+        "jest": true,
+        "cypress/globals": true
     },
     "extends": "eslint:recommended",
     "parserOptions": {

.travis.yml

@@ -1,30 +1,41 @@
 language: node_js
 node_js:
-  - "11.10.1"
+- 11.10.1
 cache:
   yarn: true
   directories:
-    - "node_modules"
+  - node_modules
+  - "~/.cache/Cypress"
+addons:
+  apt:
+    packages:
+      # Ubuntu 16+ does not install this dependency by default, so we need to install it ourselves
+      - libgconf-2-4
+    chrome: stable
 env:
+  global:
+  # this is out CYPRESS_RECORD_KEY
+  - secure: bx5SjXI+FQaPdZPOMLZ2tHI5rBeysojH2VHIbmOCVOtQYYeJO94v37ocuPM4CCBTpoM8H/ulft4lAb/AWhM1n3mndizJTZvOhjsqt6TJ+5vHnZrDoEFisgmnTb8dBA+QAGmkERRXZiLql83C1Dd+0vW3i0op7QAIY6FPz+PmSfFBDfs2XZ6WV41ZTicH84Em9y1PC+9kn0z3a32NUuwK3qM/6Q9a0h0NhBm/DblEhhOL/b95sqGFfiLvnj5Rxh4biWPmbsh09mpvHVP2AhRb4Ty/d3UCGyxD62dx2n0G7dBHzK8f3wq4zoeilhtbRoAUGPvn6L/ehlVlCefkjQzTU2X2rCeT/SR59F/5H/yjgBFS57xE0QqMPryQikEaKQPgEijX5UACzZqA7zd7uHOTbv4I1FEGrl2hShpKnNJNlJ2YIpFi9GncWhi0mYh4mS53KmEJY0IH7ZYa35q8pe8NAzPjKvsl9O1i6CtOzzub/IdtSM0RxvFEQX55iQ48Uh7QGduyRfTifFTsrUDoUtoSgc08Ze4maCmfzyIyZJzjdVa1fKYV7/NcYMYMR5OoiwvbXJ1qInVazebm3Ffp/+eDKxjXB9dJ3iANPcZgHLLzw/9Dq6tv0DIKYEcAp5phLXGEuD58Sy7Dht7tIWiXHxGZP0dZi24g43bni4fN9QDYdpY=
+  matrix:
   - TEST_SUITE=unit
   - TEST_SUITE=integration
   - TEST_SUITE=regression
   - TEST_SUITE=build
 branches:
   only:
-    - master
-    - develop
+  - master
+  - develop
 jobs:
-    include:
-      - stage: deploy
-        script: npm run build
-        deploy:
-          provider: npm
-          email:
-            secure: JINq33TBEkUPM8uTWh4XloVD8uV3uUvEoLq2Ho4ixp+a2xZBvBI10ROFQ8sCJ9YwXqe9aKRZxVuW9ePlV6yNf4cJ3qXNL4dJL0SBmxpmlG/zwxMghtj0V/mb4AUX8y7zPs3YAhjjTSzQGturAjrBn9dXQTvOY1AUyrrdCyDUt/1l2rzjylUEiOPCU1MjceM9VyasCLHR06zCIIy3Injbswp8gL2XW5CBUDeN467yyqkelMen6Q0EvA9swjw9A0wFRkQyeLEDsk1TpdNUXeYvaijOExrkepU8cpOtvgJkBpZ8qvrGVLkHOHKJQbdB7D3Uox6LNSxj5lOnNfEdpIdEKmBpybAgpf++Pi/Yuiw+NWRWCcD2vh1elndz9skOYykzlqDB8JW5oKAyGuhlXghigCJ+HycoDulCc3zRRHmoKsl5RHGcMOGWtsryFw1nUHFuM9F6awJCykwux9bxv/3HeqmYDJnDpl7hWQhFGkUG/0ygN6aaKyKFn8Nlr8d+uuZ5DaoD0KOhsNxKBuH3Js/4KryWt7LGxEdX8MyMyol5nIgHEFbh8RjRqE3iDhjxTb1MucRbbWZwiA+JIvnGUIrQhQiHvbb4dUvuHzXy9WJJ+iMGBHKVNvV3ZBxCMbiw3mFIv/0ClgJG5gG37m1oTMKTmxVf9wYxqNhM+2cBMy0wQsE=
-          api_key:
-            secure: irrK4TWdklM+pTNINlOk1EeQtT0JjPd3/9JhKDGwk/MEyAjw8+Flk7OH9MtF9gzc6AWL/8xVB05s+IyiJwcHuC/H3l4QunHl1MW343OVRG0cZzGXbJJgN9HfL+hBmOg6uOS8YLCk2ETPtYkfHuelwoPw/hIoKNnp7ZlgRV9X7Pi0qhP3ZJwHhnuGTt0TwqaGXTeUXEyycjjwBRlPf9o9UhReMxTX/7k2TEPaG0F4whkQqkcel7eZM9YB23qANMtmYfDhl1W2OxepiXpgwqtWQkj0bO9EzsTtbyEsGVbaPU+hAuZG4CDdCXyuNlL9SqNLYJdDPJuPvjMEp39/E3F+0sRiLoGu9c5Zb5v6vwnUThVhSR6EMp4W6cZihNP7PGuL+C/KqDSkPniY9qqtbCr0OasiAIgs78lfzyXMB0iXZXD9lpuKScNkSulQmQihOAIYrvfE7Z6iEnqf/v3WYytA1q+VYhP/65WOCaFMaM32qRp1XLmQFPQvACtW4LnC+OL7iRC5A3AXJrJVrib1uwl9FjaKSzqP8sThaWOW3ecipleC4ncD/O4YF2MHdH4K/4D2Xi0A0GXJ8YxvG6/pxCFgWFK+W2aMTnIzP7ATaVk91dLavFi8YC5G8zsl6dXkNHH9pFVMDvb0fLb9HMlkERhiUSV5Wo/LzBx1kWRRIH7mrh0=
-          on:
-            repo: bbc/VideoContext
-            branch: master
-          skip_cleanup: true
+  include:
+  - stage: deploy
+    script: npm run build
+    deploy:
+      provider: npm
+      email:
+        secure: JINq33TBEkUPM8uTWh4XloVD8uV3uUvEoLq2Ho4ixp+a2xZBvBI10ROFQ8sCJ9YwXqe9aKRZxVuW9ePlV6yNf4cJ3qXNL4dJL0SBmxpmlG/zwxMghtj0V/mb4AUX8y7zPs3YAhjjTSzQGturAjrBn9dXQTvOY1AUyrrdCyDUt/1l2rzjylUEiOPCU1MjceM9VyasCLHR06zCIIy3Injbswp8gL2XW5CBUDeN467yyqkelMen6Q0EvA9swjw9A0wFRkQyeLEDsk1TpdNUXeYvaijOExrkepU8cpOtvgJkBpZ8qvrGVLkHOHKJQbdB7D3Uox6LNSxj5lOnNfEdpIdEKmBpybAgpf++Pi/Yuiw+NWRWCcD2vh1elndz9skOYykzlqDB8JW5oKAyGuhlXghigCJ+HycoDulCc3zRRHmoKsl5RHGcMOGWtsryFw1nUHFuM9F6awJCykwux9bxv/3HeqmYDJnDpl7hWQhFGkUG/0ygN6aaKyKFn8Nlr8d+uuZ5DaoD0KOhsNxKBuH3Js/4KryWt7LGxEdX8MyMyol5nIgHEFbh8RjRqE3iDhjxTb1MucRbbWZwiA+JIvnGUIrQhQiHvbb4dUvuHzXy9WJJ+iMGBHKVNvV3ZBxCMbiw3mFIv/0ClgJG5gG37m1oTMKTmxVf9wYxqNhM+2cBMy0wQsE=
+      api_key:
+        secure: irrK4TWdklM+pTNINlOk1EeQtT0JjPd3/9JhKDGwk/MEyAjw8+Flk7OH9MtF9gzc6AWL/8xVB05s+IyiJwcHuC/H3l4QunHl1MW343OVRG0cZzGXbJJgN9HfL+hBmOg6uOS8YLCk2ETPtYkfHuelwoPw/hIoKNnp7ZlgRV9X7Pi0qhP3ZJwHhnuGTt0TwqaGXTeUXEyycjjwBRlPf9o9UhReMxTX/7k2TEPaG0F4whkQqkcel7eZM9YB23qANMtmYfDhl1W2OxepiXpgwqtWQkj0bO9EzsTtbyEsGVbaPU+hAuZG4CDdCXyuNlL9SqNLYJdDPJuPvjMEp39/E3F+0sRiLoGu9c5Zb5v6vwnUThVhSR6EMp4W6cZihNP7PGuL+C/KqDSkPniY9qqtbCr0OasiAIgs78lfzyXMB0iXZXD9lpuKScNkSulQmQihOAIYrvfE7Z6iEnqf/v3WYytA1q+VYhP/65WOCaFMaM32qRp1XLmQFPQvACtW4LnC+OL7iRC5A3AXJrJVrib1uwl9FjaKSzqP8sThaWOW3ecipleC4ncD/O4YF2MHdH4K/4D2Xi0A0GXJ8YxvG6/pxCFgWFK+W2aMTnIzP7ATaVk91dLavFi8YC5G8zsl6dXkNHH9pFVMDvb0fLb9HMlkERhiUSV5Wo/LzBx1kWRRIH7mrh0=
+      on:
+        repo: bbc/VideoContext
+        branch: master
+      skip_cleanup: true

Dockerfile

@@ -0,0 +1,31 @@
+# Build with:
+#   docker build -t cypress-tests .
+#
+# Run with:
+#   docker run --shm-size=256m cypress-tests
+#
+# Or, build and run with:
+#   docker build -t cypress-tests . && docker run --shm-size=256m cypress-tests
+#
+# To generate new snapshots, run:
+#   docker run -v `pwd`/test/cypress/snapshots:/cypress/test/cypress/snapshots -e "CYPRESS_updateSnapshots=true" --shm-size=256m cypress-tests
+
+# This uses the Cypress image with Chrome support
+# We need Chrome as Electron doesn't support WebGL
+FROM cypress/browsers:node11.13.0-chrome73
+
+# Define the working directory
+WORKDIR /cypress
+
+# Copy everything needed for installation
+COPY package.json yarn.lock ./
+
+# Install cypress
+RUN npm install
+
+# Copy over the source, tests, etc
+# The ".dockerignore" prevents the user's `node_modules/*` from being copied
+COPY . .
+
+# Run the CI version of cypress
+CMD ["yarn", "test-ci:cypress"]

README.md

@@ -494,7 +494,7 @@ You can view more advanced usage examples [here](AdvancedExamples.md).
 
 VideoContext has a pretty standard `package.json`
 
-```
+```sh
 # install build and development dependencies
 npm install
 
@@ -504,10 +504,15 @@ npm run dev
 # watch unit and integration tests
 npm run test-watch
 
-# run regression tests in headless browser
-npm run test-regression
+# run the end-to-end regression tests in a headless browser
+npm run cypress
 ```
 
+For more information on writing, running and debugging the end-to-end cypress tests
+see [./test/cypress#readme](./test/cypress#readme).
+
+For an overview of all testing see [./test#readme](./test#readme)
+
 ### Gitflow
 
 VideoContext uses the gitflow branching model.
@@ -534,15 +539,15 @@ CI will publish to npm when the release branch has been merged into master.
 2. `git pull`
 3. `git checkout -b release-xxx`
 4. tag and push using script
-    - `npm run release:patch|minor|major`
+   - `npm run release:patch|minor|major`
 5. open pull request against master
 6. merge when tests have passed
 7. merge master back in to develop:
-    - `git checkout master`
-    - `git pull`
-    - `git checkout develop`
-    - `git merge master`
-    - `git push`
+   - `git checkout master`
+   - `git pull`
+   - `git checkout develop`
+   - `git merge master`
+   - `git push`
 
 There is one housekeeping task (this will be automated at some point):
 

cypress-run.sh

@@ -0,0 +1,55 @@
+# This script handles the different ways of running Cypress across
+# environments.
+#
+# Ideally Cypress would be run the same way everywhere, but there is
+# a bug where the snapshots are generated at different sizes between
+# headless and headed - https://github.com/palmerhq/cypress-image-snapshot/issues/67.
+
+# Use the first argument as the switch "mode"
+# If no argument is passed, then it defaults to "ci" mode.
+case ${1} in
+    "local")
+        # Run Cypress locally with the headed browser (GUI)
+        CYPRESS_local=true ./node_modules/.bin/cypress open
+        ;;
+    "local-update-snapshots")
+        # Update snapshots for the local Cypress GUI version.
+        # These are typically larger (see bug comment above) than the
+        # versions generated and used by Travis (CI) and Docker.
+        CYPRESS_local=true CYPRESS_updateSnapshots=true ./node_modules/.bin/cypress run --browser chrome
+        ;;
+    "docker")
+        # Run Cypress in a headless docker image.
+        # This environment is closer to the Travis (CI) environment.
+        docker build -t videocontext-cypress-tests .
+        # Run with:
+        #   --shm-size: Increased shared memory size, otherwise the tests
+        #               sometimes hang.
+        #   -v: Mounts the host snapshot directory, so any generated snapshots
+        #       will be created/modified in the user's filesystem ready to be committed.
+        docker run -v `pwd`/test/cypress/snapshots:/cypress/test/cypress/snapshots --shm-size=256m videocontext-cypress-tests
+        ;;
+    "docker-update-snapshots")
+        # Generate new snapshots using the headless docker image.
+        # These snapshots should be identical to those generated by Travis (CI)
+        docker build -t videocontext-cypress-tests .
+        # Run with:
+        #   --shm-size: Increased shared memory size, otherwise the tests
+        #               sometimes hang.
+        #   -v: Mounts the host snapshot directory, so any generated snapshots
+        #       will be created/modified in the user's filesystem ready to be committed.
+        docker run -v `pwd`/test/cypress/snapshots:/cypress/test/cypress/snapshots -e "CYPRESS_updateSnapshots=true" --shm-size=256m videocontext-cypress-tests
+        ;;
+    *)
+        # Perform the command `cypress run` record the run if `CYPRESS_RECORD_KEY`
+        # is set this way we'll only CI will record runs and no non-robots require
+        # the record key.
+        if [ -z "$CYPRESS_RECORD_KEY" ]
+        then
+            ./node_modules/.bin/cypress run --browser chrome
+        else
+            # use chrome on CI so that WebGL works
+            ./node_modules/.bin/cypress run --record --browser chrome
+        fi
+        ;;
+esac

cypress.json

@@ -0,0 +1,10 @@
+{
+  "fixturesFolder": "test/cypress/fixtures",
+  "integrationFolder": "test/cypress/integration",
+  "pluginsFile": "test/cypress/plugins/index.js",
+  "screenshotsFolder": "test/cypress/screenshots",
+  "videosFolder": "test/cypress/videos",
+  "supportFile": "test/cypress/support/index.js",
+  "baseUrl": "http://localhost:3001/test/cypress/html/",
+  "projectId": "mdase9"
+}

package.json

@@ -30,12 +30,21 @@
     "test-integration": "jest test/integration/*.js",
     "test-watch": "jest --watch",
     "test-coverage": "jest test/unit/*.js --coverage --collectCoverageFrom 'src/**/*.js'",
-    "test-regression": "jest test/regression/*.js",
+    "test-ci:cypress": "yarn run _server_cy_then _cy:run",
+    "cypress": "./cypress-run.sh docker",
+    "cypress:update-snapshots": "./cypress-run.sh docker-update-snapshots",
+    "cypress:gui": "yarn run _server_cy_then _cy:open",
+    "cypress:gui-update-snapshots": "yarn run _server_cy_then _cy:local-update-snapshots",
+    "_cy:local-update-snapshots": "./cypress-run.sh local-update-snapshots",
+    "_cy:open": "./cypress-run.sh local",
+    "_cy:run": "./cypress-run.sh",
+    "_cy-server": "concurrently \"npm run dev\" \"ws -p 3001\"",
+    "_server_cy_then": "start-server-and-test _cy-server http://localhost:3001",
     "docs": "bash ./build-docs.sh",
     "docs-commit": "npm run docs && git add docs && git diff-index --quiet HEAD || git commit -m 'update docs in preparation for release'",
     "format": "prettier --write '{src,test}/**/*.js'",
     "lint": "eslint '{src,test}/**/*.js'",
-    "precommit": "npm run lint && npm run format && npm run test-unit",
+    "precommit": "npm run lint && npm run format",
     "release:major": "npm run docs-commit && changelog -M && git add CHANGELOG.md && git commit -m 'updated CHANGELOG.md' && npm version --new-version major",
     "release:minor": "npm run docs-commit && changelog -m && git add CHANGELOG.md && git commit -m 'updated CHANGELOG.md' && npm version --new-version minor",
     "release:patch": "npm run docs-commit && changelog -p && git add CHANGELOG.md && git commit -m 'updated CHANGELOG.md' && npm version --new-version patch",
@@ -52,20 +61,24 @@
     "babel-plugin-add-module-exports": "^0.2.1",
     "babel-plugin-transform-es2015-modules-umd": "^6.24.1",
     "babel-preset-env": "^1.7.0",
+    "concurrently": "^4.1.1",
+    "cypress": "3.3.2",
+    "cypress-image-snapshot": "^3.0.0",
     "eslint": "^4.18.2",
     "eslint-loader": "^2.0.0",
+    "eslint-plugin-cypress": "^2.2.0",
     "generate-changelog": "^1.7.1",
     "http-server": "^0.11.1",
     "husky": "^0.14.3",
     "jest": "^24.3.1",
     "jest-cli": "^23.1.0",
     "jest-image-snapshot": "^2.4.2",
     "jsdoc": "^3.4.3",
-    "path": "^0.12.7",
+    "local-web-server": "^2.6.1",
     "prettier": "^1.13.2",
-    "puppeteer": "1.11.0",
     "raw-loader": "^0.5.1",
     "sinon": "^7.2.7",
+    "start-server-and-test": "^1.7.11",
     "webgl-mock": "^0.1.7",
     "webpack": "^4.10.2",
     "webpack-cli": "^3.0.1"

test.sh

@@ -23,7 +23,7 @@ elif [ "$TEST_SUITE" == "regression" ]
       echo "# Running Available Regression Tests #"
       echo "#####################################"
 
-      npm run test-regression
+      npm run test-ci:cypress
 
 elif [ "$TEST_SUITE" == "build" ]
   then

test/README.md

@@ -0,0 +1,34 @@
+# Test
+
+![build status](https://travis-ci.org/bbc/VideoContext.svg?branch=master)
+![cypress](https://img.shields.io/badge/testing-cypress-informational.svg)
+![jest](https://img.shields.io/badge/testing-jest-informational.svg?style=flat&logo=jest)
+
+There are three approaches to testing within `VideoContext`. All must pass on CI for PRs to be merged.
+
+### [🔗unit](./unit)
+
+Testing the individual components that make up `VideoContext` as isolated units
+
+- **framework**: [jest](https://jestjs.io/)
+- **coverage**: moderate
+- **coupling**: high - we're using a reasonable amount of mocking and testing against some private APIs
+
+### [🔗integration](./integration)
+
+Spec testing on the public APIs of `VideoContext`
+
+- **framework**: [jest](https://jestjs.io/)
+- **coverage**: moderate/low
+- **coupling**: low
+
+### [🔗cypress](./cypress)
+
+End-to-end regression testing in the browser. `VideoContext` is driven as it would be in the real world.
+Cypress is used to take and compare screenshots out the output to previous builds
+
+- **framework**: [cypress](https://www.cypress.io/)
+- **coverage**: low
+- **coupling**: low
+
+For more detail see [./cypress#readme](./cypress#readme)

test/cypress/.gitignore

@@ -0,0 +1,3 @@
+__diff_output__
+screenshots
+videos