From 25db3dc0e21905ea8d7c2d3c2d2a6a338a1b399b Mon Sep 17 00:00:00 2001 From: Andrew Duthie Date: Tue, 13 Oct 2020 10:41:01 -0400 Subject: [PATCH 01/10] Add basic visual regression testing --- package-lock.json | 300 ++++++++++++++++++++++++++++++++++++++-- package.json | 6 +- test/screenshot.test.js | 130 +++++++++++++++++ 3 files changed, 420 insertions(+), 16 deletions(-) create mode 100644 test/screenshot.test.js diff --git a/package-lock.json b/package-lock.json index c91b47a6..b18ea4cd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2010,6 +2010,23 @@ "lodash": "^4.17.4", "mkdirp": "^0.5.1", "source-map-support": "^0.4.15" + }, + "dependencies": { + "minimist": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", + "dev": true + }, + "mkdirp": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", + "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", + "dev": true, + "requires": { + "minimist": "^1.2.5" + } + } } }, "babel-runtime": { @@ -2466,6 +2483,21 @@ "xtend": "^4.0.0" }, "dependencies": { + "minimist": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", + "dev": true + }, + "mkdirp": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", + "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", + "dev": true, + "requires": { + "minimist": "^1.2.5" + } + }, "punycode": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", @@ -5229,6 +5261,21 @@ "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", "dev": true }, + "minimist": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", + "dev": true + }, + "mkdirp": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", + "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", + "dev": true, + "requires": { + "minimist": "^1.2.5" + } + }, "ms": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", @@ -5881,6 +5928,23 @@ "debug": "2.6.9", "mkdirp": "0.5.1", "yauzl": "2.4.1" + }, + "dependencies": { + "minimist": { + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", + "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=", + "dev": true + }, + "mkdirp": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", + "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", + "dev": true, + "requires": { + "minimist": "0.0.8" + } + } } }, "extsprintf": { @@ -6891,6 +6955,23 @@ "inherits": "~2.0.0", "mkdirp": ">=0.5 0", "rimraf": "2" + }, + "dependencies": { + "minimist": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", + "dev": true + }, + "mkdirp": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", + "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", + "dev": true, + "requires": { + "minimist": "^1.2.5" + } + } } }, "function-bind": { @@ -7836,6 +7917,21 @@ "supports-color": "^5.3.0" } }, + "minimist": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", + "dev": true + }, + "mkdirp": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", + "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", + "dev": true, + "requires": { + "minimist": "^1.2.5" + } + }, "plugin-error": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/plugin-error/-/plugin-error-1.0.1.tgz", @@ -9035,6 +9131,23 @@ "js-yaml": "^3.7.0", "mkdirp": "^0.5.1", "once": "^1.4.0" + }, + "dependencies": { + "minimist": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", + "dev": true + }, + "mkdirp": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", + "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", + "dev": true, + "requires": { + "minimist": "^1.2.5" + } + } } }, "istanbul-lib-coverage": { @@ -9085,6 +9198,21 @@ "integrity": "sha1-nZ55MWXOAXoA8AQYxD+UKnsdEfo=", "dev": true }, + "minimist": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", + "dev": true + }, + "mkdirp": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", + "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", + "dev": true, + "requires": { + "minimist": "^1.2.5" + } + }, "supports-color": { "version": "3.2.3", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-3.2.3.tgz", @@ -9118,6 +9246,21 @@ "ms": "^2.1.1" } }, + "minimist": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", + "dev": true + }, + "mkdirp": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", + "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", + "dev": true, + "requires": { + "minimist": "^1.2.5" + } + }, "ms": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", @@ -10367,6 +10510,21 @@ "supports-color": "^5.3.0" } }, + "minimist": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", + "dev": true + }, + "mkdirp": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", + "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", + "dev": true, + "requires": { + "minimist": "^1.2.5" + } + }, "supports-color": { "version": "5.5.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", @@ -10414,6 +10572,21 @@ "supports-color": "^5.3.0" } }, + "minimist": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", + "dev": true + }, + "mkdirp": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", + "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", + "dev": true, + "requires": { + "minimist": "^1.2.5" + } + }, "slash": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/slash/-/slash-1.0.0.tgz", @@ -11501,21 +11674,10 @@ } }, "mkdirp": { - "version": "0.5.1", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", - "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", - "dev": true, - "requires": { - "minimist": "0.0.8" - }, - "dependencies": { - "minimist": { - "version": "0.0.8", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", - "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=", - "dev": true - } - } + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "dev": true }, "module-deps": { "version": "6.2.1", @@ -11644,6 +11806,21 @@ "which": "1" }, "dependencies": { + "minimist": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", + "dev": true + }, + "mkdirp": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", + "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", + "dev": true, + "requires": { + "minimist": "^1.2.5" + } + }, "semver": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/semver/-/semver-5.3.0.tgz", @@ -11721,6 +11898,21 @@ "integrity": "sha1-uWjGsKBDhDJJAui/Gl3zJXmkUP4=", "dev": true }, + "minimist": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", + "dev": true + }, + "mkdirp": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", + "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", + "dev": true, + "requires": { + "minimist": "^1.2.5" + } + }, "nan": { "version": "2.14.0", "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.0.tgz", @@ -12554,6 +12746,23 @@ "pinkie": "^2.0.0" } }, + "pixelmatch": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/pixelmatch/-/pixelmatch-5.2.1.tgz", + "integrity": "sha512-WjcAdYSnKrrdDdqTcVEY7aB7UhhwjYQKYhHiBXdJef0MOaQeYpUdQ+iVyBLa5YBKS8MPVPPMX7rpOByISLpeEQ==", + "dev": true, + "requires": { + "pngjs": "^4.0.1" + }, + "dependencies": { + "pngjs": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-4.0.1.tgz", + "integrity": "sha512-rf5+2/ioHeQxR6IxuYNYGFytUyG3lma/WW1nsmjeHlWwtb2aByla6dkVc8pmJ9nplzkTA0q2xx7mMWrOTqT4Gg==", + "dev": true + } + } + }, "pkg-dir": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-2.0.0.tgz", @@ -12632,6 +12841,12 @@ "integrity": "sha512-2qHaIQr2VLRFoxe2nASzsV6ef4yOOH+Fi9FBOVH6cqeSgUnoyySPZkxzLuzd+RYOQTRpROA0ztTMqxROKSb/nA==", "dev": true }, + "pngjs": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-5.0.0.tgz", + "integrity": "sha512-40QW5YalBNfQo5yRYmiw7Yz6TKKVr3h6970B2YE+3fQpsWcrbj1PzJgxeJ19DRQjhMbKPIuMY8rFaXc8moolVw==", + "dev": true + }, "posix-character-classes": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/posix-character-classes/-/posix-character-classes-0.1.1.tgz", @@ -15477,6 +15692,21 @@ "supports-color": "^5.3.0" } }, + "minimist": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", + "dev": true + }, + "mkdirp": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", + "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", + "dev": true, + "requires": { + "minimist": "^1.2.5" + } + }, "supports-color": { "version": "5.5.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", @@ -15577,6 +15807,23 @@ "yallist": "^3.0.3" }, "dependencies": { + "minimist": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", + "dev": true, + "optional": true + }, + "mkdirp": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", + "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", + "dev": true, + "optional": true, + "requires": { + "minimist": "^1.2.5" + } + }, "safe-buffer": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.0.tgz", @@ -16995,6 +17242,23 @@ "dev": true, "requires": { "mkdirp": "^0.5.1" + }, + "dependencies": { + "minimist": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", + "dev": true + }, + "mkdirp": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", + "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", + "dev": true, + "requires": { + "minimist": "^1.2.5" + } + } } }, "write-file-atomic": { @@ -17047,6 +17311,12 @@ "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=", "dev": true }, + "yaml": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-1.10.0.tgz", + "integrity": "sha512-yr2icI4glYaNG+KWONODapy2/jDdMSDnrONSjblABjD9B4Z5LgiircSt8m8sRZFNi08kG9Sm0uSHtEmP3zaEGg==", + "dev": true + }, "yargs": { "version": "11.1.1", "resolved": "https://registry.npmjs.org/yargs/-/yargs-11.1.1.tgz", diff --git a/package.json b/package.json index cbe0160e..cd347430 100644 --- a/package.json +++ b/package.json @@ -62,8 +62,11 @@ "gulp-uglify": "^3.0.2", "jest": "^23.6.0", "jest-puppeteer": "^3.9.1", + "mkdirp": "^1.0.4", "pa11y-ci": "^2.3.0", "path": "^0.12.7", + "pixelmatch": "^5.2.1", + "pngjs": "^5.0.0", "puppeteer": "^1.18.1", "serve": "^10.1.2", "stylelint": "^9.10.1", @@ -72,6 +75,7 @@ "uswds-gulp": "github:uswds/uswds-gulp", "vinyl-buffer": "^1.0.1", "vinyl-source-stream": "^2.0.0", - "wait-on": "^3.3.0" + "wait-on": "^3.3.0", + "yaml": "^1.10.0" } } diff --git a/test/screenshot.test.js b/test/screenshot.test.js new file mode 100644 index 00000000..76638714 --- /dev/null +++ b/test/screenshot.test.js @@ -0,0 +1,130 @@ +/* eslint-disable no-restricted-syntax, no-await-in-loop, no-param-reassign */ + +const { promises: fsPromises, readFileSync } = require('fs'); +const { join } = require('path'); +const { spawnSync } = require('child_process'); +const assert = require('assert'); +const mkdirp = require('mkdirp'); +const { PNG } = require('pngjs'); +const match = require('pixelmatch'); +const YAML = require('yaml'); + +const { writeFile } = fsPromises; + +const ACKNOWLEDGED_COMMIT = '247b7e6815b358d7962ad2bc21d34dd9122bbe69'; +const LOCAL_HOST = `http://localhost:${process.env.JEST_PORT}`; +const REMOTE_HOST = 'https://design.login.gov'; +const DIFF_DIRECTORY = join(__dirname, '../tmp/results/screenshot-diff'); +const { url: URL_PREFIX, github_repo_url: GITHUB_REPO_URL } = YAML.parse( + readFileSync(join(__dirname, '../_config.yml'), 'utf8'), +); + +async function getURLsFromSitemap(url) { + await page.goto(url); + return page.$$eval('url loc', locs => locs.map(loc => loc.textContent)); +} + +async function stubAnimations() { + await page.evaluate(() => { + const isGif = img => img.src.endsWith('.gif'); + function stubGif(img) { + img.width = img.width; + img.height = img.height; + img.src = 'data:image/gif;base64,R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw=='; + img.removeAttribute('srcset'); + } + + [...document.querySelectorAll('img')].filter(isGif).forEach(stubGif); + }); +} + +function getLatestCommit() { + const [sha] = spawnSync('git', ['ls-remote', GITHUB_REPO_URL, 'HEAD']) + .stdout + .toString() + .trim() + .split('\t'); + + return sha; +} + +async function getScreenshot(url) { + await page.goto(url); + await stubAnimations(); + return page.screenshot({ fullPage: true }); +} + +function getURLPath(url) { + const prefix = new URL(URL_PREFIX).pathname; + const { pathname } = new URL(url); + return pathname.indexOf(prefix) === 0 ? pathname.slice(prefix.length) : pathname; +} + +function getDiffOutputBaseFileName(pathname) { + let normalizedPathname = pathname; + normalizedPathname = normalizedPathname.replace(/^\/|\/$/g, ''); + normalizedPathname = normalizedPathname.replace(/\W/g, '_'); + normalizedPathname = normalizedPathname || 'home'; + + return join(DIFF_DIRECTORY, normalizedPathname); +} + +function fillImageToSize(image, width, height) { + if (image.width === width && image.height === height) { + return image; + } + + const resizedImage = new PNG({ width, height, fill: true }); + PNG.bitblt(image, resizedImage, 0, 0, image.width, image.height, 0, 0); + + for (let y = image.height; y < height; y += 1) { + for (let x = image.width; x < width; x += 1) { + // eslint-disable-next-line no-bitwise + const index = ((resizedImage.width * y) + x) << 2; + resizedImage.data[index] = 0; + resizedImage.data[index + 1] = 0; + resizedImage.data[index + 2] = 0; + resizedImage.data[index + 3] = 64; + } + } + + return resizedImage; +} + +const testFn = getLatestCommit() === ACKNOWLEDGED_COMMIT ? test.skip : test; + +testFn('screenshot visual regression', async () => { + const paths = (await getURLsFromSitemap(`${LOCAL_HOST}/sitemap.xml`)).map(getURLPath); + + for (const path of paths) { + const local = await getScreenshot(LOCAL_HOST + path); + const remote = await getScreenshot(REMOTE_HOST + path); + const localPNG = PNG.sync.read(local); + const remotePNG = PNG.sync.read(remote); + const width = Math.max(localPNG.width, remotePNG.width); + const height = Math.max(localPNG.height, remotePNG.height); + const resizedLocalPNG = fillImageToSize(localPNG, width, height); + const resizedRemotePNG = fillImageToSize(remotePNG, width, height); + const diff = new PNG({ width, height }); + const diffs = match( + resizedLocalPNG.data, + resizedRemotePNG.data, + diff.data, + width, + height, + { + threshold: 0.2, + }, + ); + if (diffs > 0) { + const diffOutputBase = getDiffOutputBaseFileName(path); + await mkdirp(DIFF_DIRECTORY); + await Promise.all([ + writeFile(`${diffOutputBase}-local.png`, PNG.sync.write(resizedLocalPNG)), + writeFile(`${diffOutputBase}-remote.png`, PNG.sync.write(resizedRemotePNG)), + writeFile(`${diffOutputBase}-diff.png`, PNG.sync.write(diff)), + ]); + } + assert.strictEqual(diffs, 0, `Expected "${path}" to have 0 diffs`); + } +}, 1000000); From ed514d3a39043b2e312896e203975f23b035ea80 Mon Sep 17 00:00:00 2001 From: Andrew Duthie Date: Tue, 13 Oct 2020 15:19:35 -0400 Subject: [PATCH 02/10] Add inline comments explaining RGBA Co-Authored-By: Zach Margolis --- test/screenshot.test.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/screenshot.test.js b/test/screenshot.test.js index 76638714..e43748e8 100644 --- a/test/screenshot.test.js +++ b/test/screenshot.test.js @@ -81,10 +81,10 @@ function fillImageToSize(image, width, height) { for (let x = image.width; x < width; x += 1) { // eslint-disable-next-line no-bitwise const index = ((resizedImage.width * y) + x) << 2; - resizedImage.data[index] = 0; - resizedImage.data[index + 1] = 0; - resizedImage.data[index + 2] = 0; - resizedImage.data[index + 3] = 64; + resizedImage.data[index] = 255; // Red + resizedImage.data[index + 1] = 255; // Green + resizedImage.data[index + 2] = 255; // Blue + resizedImage.data[index + 3] = 255; // Alpha (Opacity) } } From b7e910f0c9ea940e14f5b69c86728ee895b9d789 Mon Sep 17 00:00:00 2001 From: Andrew Duthie Date: Tue, 13 Oct 2020 15:20:49 -0400 Subject: [PATCH 03/10] Readable-ize assertion error message --- test/screenshot.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/screenshot.test.js b/test/screenshot.test.js index e43748e8..052913ff 100644 --- a/test/screenshot.test.js +++ b/test/screenshot.test.js @@ -125,6 +125,6 @@ testFn('screenshot visual regression', async () => { writeFile(`${diffOutputBase}-diff.png`, PNG.sync.write(diff)), ]); } - assert.strictEqual(diffs, 0, `Expected "${path}" to have 0 diffs`); + assert.strictEqual(diffs, 0, `Expected "${path}" to visually match the live site.`); } }, 1000000); From 1cddbd055802566cea426216545bedc65c495383 Mon Sep 17 00:00:00 2001 From: Andrew Duthie Date: Tue, 13 Oct 2020 16:28:46 -0400 Subject: [PATCH 04/10] Use remote host for sitemap --- test/screenshot.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/screenshot.test.js b/test/screenshot.test.js index 052913ff..35968715 100644 --- a/test/screenshot.test.js +++ b/test/screenshot.test.js @@ -94,7 +94,7 @@ function fillImageToSize(image, width, height) { const testFn = getLatestCommit() === ACKNOWLEDGED_COMMIT ? test.skip : test; testFn('screenshot visual regression', async () => { - const paths = (await getURLsFromSitemap(`${LOCAL_HOST}/sitemap.xml`)).map(getURLPath); + const paths = (await getURLsFromSitemap(`${REMOTE_HOST}/sitemap.xml`)).map(getURLPath); for (const path of paths) { const local = await getScreenshot(LOCAL_HOST + path); From 9397ea20ce245b7d7bc60e1db4f9706886ef2bcf Mon Sep 17 00:00:00 2001 From: Andrew Duthie Date: Wed, 14 Oct 2020 09:03:03 -0400 Subject: [PATCH 05/10] Split CircleCI jobs to isolate visual regression tests --- .circleci/config.yml | 62 +++++++++++++++++++++++++++++++++++++------- jest.config.js | 10 ++++++- 2 files changed, 61 insertions(+), 11 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1757e1f7..d68c98da 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,13 +1,8 @@ version: 2 -jobs: - build: - working_directory: ~/identity-style-guide - docker: - - image: circleci/ruby:2.5.3-node-browsers - +commands: + bundle-yarn-install: steps: - - checkout - restore_cache: key: dependency-cache-v1-{{ checksum "package-lock.json" }}-{{ checksum "Gemfile.lock" }} - run: @@ -21,15 +16,62 @@ jobs: paths: - ~/.npm - ~/.bundler - - run: - name: Lint JavaScript and Sass - command: npm run lint + build: + steps: - run: name: Build assets and site command: npm run build + +jobs: + lints: + working_directory: ~/identity-style-guide + docker: + - image: circleci/ruby:2.5.3-node-browsers + steps: + - checkout + - bundle-yarn-install + - run: + name: Lint JavaScript and Sass + command: npm run lint + integration: + working_directory: ~/identity-style-guide + docker: + - image: circleci/ruby:2.5.3-node-browsers + environment: + SKIP_VISUAL_REGRESSION_TEST: true + steps: + - checkout + - bundle-yarn-install + - build - run: name: Run jest integration test command: npm run test-jest + visual-regression: + working_directory: ~/identity-style-guide + docker: + - image: circleci/ruby:2.5.3-node-browsers + environment: + ONLY_VISUAL_REGRESSION_TEST: true + steps: + - checkout + - bundle-yarn-install + - build + - run: + name: Run visual regression test + command: npm run test-jest + - store_artifacts: + path: tmp/results + destination: results + - store_test_results: + path: tmp/results + accessibility: + working_directory: ~/identity-style-guide + docker: + - image: circleci/ruby:2.5.3-node-browsers + steps: + - checkout + - bundle-yarn-install + - build - run: name: Run pa11y accessibility test command: npm run test-pa11y diff --git a/jest.config.js b/jest.config.js index 0e4ffa3f..3a3e0305 100644 --- a/jest.config.js +++ b/jest.config.js @@ -5,5 +5,13 @@ module.exports = { errorOnDeprecated: true, notify: true, preset: 'jest-puppeteer', - testMatch: ['**/test/**/*.test.js'], + testMatch: [ + 'ONLY_VISUAL_REGRESSION_TEST' in process.env + ? '**/test/screenshot.test.js' + : '**/test/**/*.test.js', + ], + testPathIgnorePatterns: [ + '/node_modules/', + 'SKIP_VISUAL_REGRESSION_TEST' in process.env && 'screenshot.test.js', + ].filter(Boolean), }; From 1274cc2dfbd13c572f75be0efaec08cd7df7ed13 Mon Sep 17 00:00:00 2001 From: Andrew Duthie Date: Wed, 14 Oct 2020 09:15:46 -0400 Subject: [PATCH 06/10] Add CircleCI workflows configuration Required when not using singular "build" job --- .circleci/config.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index d68c98da..a037d91d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -80,3 +80,12 @@ jobs: destination: results - store_test_results: path: tmp/results + +workflows: + version: 2 + test: + jobs: + - lints + - integration + - visual-regression + - accessibility From 49d1a19496df825b3983d7ae78a788ae91f513c9 Mon Sep 17 00:00:00 2001 From: Andrew Duthie Date: Wed, 14 Oct 2020 09:20:53 -0400 Subject: [PATCH 07/10] Upgrade CircleCI configuration to 2.1 Necessary for "commands" support See: https://discuss.circleci.com/t/circleci-2-1-config-overview/26057 --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a037d91d..c2940672 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,4 +1,4 @@ -version: 2 +version: 2.1 commands: bundle-yarn-install: From 3b90a276469cba941ddf4c2ac88bd28c3416a4f8 Mon Sep 17 00:00:00 2001 From: Andrew Duthie Date: Wed, 14 Oct 2020 09:21:11 -0400 Subject: [PATCH 08/10] Rename bundle-yarn-install to bundle-npm-install Style guide not (currently) using Yarn --- .circleci/config.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c2940672..5554a0a0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,7 +1,7 @@ version: 2.1 commands: - bundle-yarn-install: + bundle-npm-install: steps: - restore_cache: key: dependency-cache-v1-{{ checksum "package-lock.json" }}-{{ checksum "Gemfile.lock" }} @@ -29,7 +29,7 @@ jobs: - image: circleci/ruby:2.5.3-node-browsers steps: - checkout - - bundle-yarn-install + - bundle-npm-install - run: name: Lint JavaScript and Sass command: npm run lint @@ -41,7 +41,7 @@ jobs: SKIP_VISUAL_REGRESSION_TEST: true steps: - checkout - - bundle-yarn-install + - bundle-npm-install - build - run: name: Run jest integration test @@ -54,7 +54,7 @@ jobs: ONLY_VISUAL_REGRESSION_TEST: true steps: - checkout - - bundle-yarn-install + - bundle-npm-install - build - run: name: Run visual regression test @@ -70,7 +70,7 @@ jobs: - image: circleci/ruby:2.5.3-node-browsers steps: - checkout - - bundle-yarn-install + - bundle-npm-install - build - run: name: Run pa11y accessibility test From 0bbb5a33c20d93b618b0ced375f73b3f72d7bd92 Mon Sep 17 00:00:00 2001 From: Andrew Duthie Date: Wed, 14 Oct 2020 09:29:10 -0400 Subject: [PATCH 09/10] Add README documentation for visual regression tests --- README.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 8448a4b5..844b6ca2 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,12 @@ npm start npm run lint ``` +## Visual regression tests + +When a pull request is submitted, a visual regression test will be automatically run to check for any visual changes between the working copy of the branch and the live documentation site. These will be reported as the `ci/circleci: visual-regression` GitHub status check. + +A failure of this status check only indicates that a visual change was detected. Depending on the types of changes being proposed, this may be expected. Anyone with access to the CircleCI dashboard can review the specific changes by following the status check "Details" link and comparing the set of screenshots under the "Artifacts" tab. If the visual changes are acceptable, the pull request can be merged, even if the status check is reported as a failure. + ## Deploying documentation updates Documentation deploys are performed automatically upon merging to `master` by [Federalist](https://federalist.18f.gov/). Federalist performs the following steps: @@ -44,7 +50,7 @@ More information can be found in Federalist’s [How Builds Work](https://federa ## Publishing a release to `npm` -When you're ready to release a new version of the `identity-style-guide` package there are just a few steps to take. +When you're ready to release a new version of the `identity-style-guide` package there are just a few steps to take. 1️⃣ Make sure all the changes indended for release are merged into the `master` branch. @@ -54,7 +60,7 @@ When you're ready to release a new version of the `identity-style-guide` package `npm version patch -m "Upgrade to %s for reasons"` -And a new version will be created. +And a new version will be created. 4️⃣ Once you’re satisfied with any updates, do a trial publish to `npm` by running: @@ -70,4 +76,4 @@ No need to run any special build steps — the publish script will lint the sour npm publish ``` -6️⃣ Document the release on Github. After you've pushed the release changes back up to `master`, [create a new release](https://github.com/18F/identity-style-guide/releases) with a target of `master`. The release version should match the version you just sent off to `npm` (like `v2.1.5`) and the title can be the same. Use the release notes to link to any important issues or pull requests that were addressed in the release. +6️⃣ Document the release on Github. After you've pushed the release changes back up to `master`, [create a new release](https://github.com/18F/identity-style-guide/releases) with a target of `master`. The release version should match the version you just sent off to `npm` (like `v2.1.5`) and the title can be the same. Use the release notes to link to any important issues or pull requests that were addressed in the release. From 669cf2c0b5d171e79f377b16a3660f42b8d5e568 Mon Sep 17 00:00:00 2001 From: Andrew Duthie Date: Wed, 14 Oct 2020 09:52:02 -0400 Subject: [PATCH 10/10] Remove acknowledged commit logic New workflow will be to bypass failed result of visual regression test status check as implied acknowledgment of changes --- test/screenshot.test.js | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/test/screenshot.test.js b/test/screenshot.test.js index 35968715..275e2f43 100644 --- a/test/screenshot.test.js +++ b/test/screenshot.test.js @@ -2,7 +2,6 @@ const { promises: fsPromises, readFileSync } = require('fs'); const { join } = require('path'); -const { spawnSync } = require('child_process'); const assert = require('assert'); const mkdirp = require('mkdirp'); const { PNG } = require('pngjs'); @@ -11,13 +10,10 @@ const YAML = require('yaml'); const { writeFile } = fsPromises; -const ACKNOWLEDGED_COMMIT = '247b7e6815b358d7962ad2bc21d34dd9122bbe69'; const LOCAL_HOST = `http://localhost:${process.env.JEST_PORT}`; const REMOTE_HOST = 'https://design.login.gov'; const DIFF_DIRECTORY = join(__dirname, '../tmp/results/screenshot-diff'); -const { url: URL_PREFIX, github_repo_url: GITHUB_REPO_URL } = YAML.parse( - readFileSync(join(__dirname, '../_config.yml'), 'utf8'), -); +const { url: URL_PREFIX } = YAML.parse(readFileSync(join(__dirname, '../_config.yml'), 'utf8')); async function getURLsFromSitemap(url) { await page.goto(url); @@ -38,16 +34,6 @@ async function stubAnimations() { }); } -function getLatestCommit() { - const [sha] = spawnSync('git', ['ls-remote', GITHUB_REPO_URL, 'HEAD']) - .stdout - .toString() - .trim() - .split('\t'); - - return sha; -} - async function getScreenshot(url) { await page.goto(url); await stubAnimations(); @@ -91,9 +77,7 @@ function fillImageToSize(image, width, height) { return resizedImage; } -const testFn = getLatestCommit() === ACKNOWLEDGED_COMMIT ? test.skip : test; - -testFn('screenshot visual regression', async () => { +test('screenshot visual regression', async () => { const paths = (await getURLsFromSitemap(`${REMOTE_HOST}/sitemap.xml`)).map(getURLPath); for (const path of paths) {