Initial commit adding a11y docker container and a11y tests

wikimedia · Aug 11, 2023 · 36f1a25 · 36f1a25
1 parent 3bce283
commit 36f1a25
Show file tree

Hide file tree

Showing 11 changed files with 623 additions and 53 deletions.
diff --git a/.eslintignore b/.eslintignore
@@ -1,4 +1,5 @@
 report
+report-a11y
 package-lock.json
 coverage
 context.json
diff --git a/Dockerfile.a11y-regression b/Dockerfile.a11y-regression
@@ -0,0 +1,53 @@
+FROM node:16.15.1-bullseye
+
+ARG PA11Y_VERSION
+
+ENV \
+	PA11Y_VERSION=$PA11Y_VERSION
+
+# Base packages
+RUN apt-get update && \
+	apt-get install -y git sudo software-properties-common
+
+# Find links to Chromium arm64 binaries here: http://snapshot.debian.org/binary/chromium/
+ENV CHROMIUM_ARM_URL http://snapshot.debian.org/archive/debian-security/20220902T180902Z/pool/updates/main/c/chromium/
+ENV CHROMIUM_ARM_BINARY chromium_105.0.5195.52-1~deb11u1_arm64.deb
+ENV CHROMIUM_ARM_COMMON_BINARY chromium-common_105.0.5195.52-1~deb11u1_arm64.deb
+
+RUN /bin/bash -c 'set -ex && \
+    ARCH=`uname -m` && \
+    if [ "$ARCH" == "x86_64" ]; then \
+       sudo npm install -g --unsafe-perm=true --allow-root pa11y@${PA11Y_VERSION}; \
+    else \
+       sudo PUPPETEER_SKIP_DOWNLOAD=true PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true npm install -g --unsafe-perm=true --allow-root pa11y@${PA11Y_VERSION} && \
+       wget "${CHROMIUM_ARM_URL}${CHROMIUM_ARM_COMMON_BINARY}" \
+            "${CHROMIUM_ARM_URL}${CHROMIUM_ARM_BINARY}" && \
+       apt install -y "./${CHROMIUM_ARM_COMMON_BINARY}" \
+                      "./${CHROMIUM_ARM_BINARY}" && \
+       sudo test -f /usr/bin/chromium && sudo ln -s /usr/bin/chromium /usr/bin/chromium-browser && sudo ln -s /usr/bin/chromium /usr/bin/chrome; \
+    fi'
+
+RUN sudo npm install -g mustache
+
+RUN wget https://dl-ssl.google.com/linux/linux_signing_key.pub && sudo apt-key add linux_signing_key.pub
+RUN sudo add-apt-repository "deb http://dl.google.com/linux/chrome/deb/ stable main"
+
+RUN apt-get -qqy update \
+  && apt-get -qqy --no-install-recommends install \
+    libxshmfence-dev \
+    libfontconfig \
+    libfreetype6 \
+    xfonts-cyrillic \
+    xfonts-scalable \
+    fonts-liberation \
+    fonts-ipafont-gothic \
+    fonts-wqy-zenhei \
+    libgbm-dev \
+    gconf-service libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxss1 libxtst6 libappindicator1 libnss3 libasound2 libatk1.0-0 libc6 ca-certificates fonts-liberation lsb-release xdg-utils wget \
+  && rm -rf /var/lib/apt/lists/* \
+  && apt-get -qyy clean
+
+ENV NODE_PATH=/usr/local/lib/node_modules
+
+#ENTRYPOINT ["tail", "-f", "/dev/null"]
+ENTRYPOINT ["node", "src/a11y/runA11yTests.js"]
diff --git a/configDesktopA11y.js b/configDesktopA11y.js
@@ -0,0 +1,47 @@
+// @ts-nocheck
+const utils = require( './utils' );
+
+const NAMESPACE = 'desktop';
+const BASE_URL = process.env.PIXEL_MW_SERVER;
+
+const testDefaults = {
+	viewport: {
+		width: 1200,
+		height: 1080
+	},
+	runners: [
+		'axe',
+		'htmlcs'
+	],
+	includeWarnings: true,
+	includeNotices: true,
+	ignore: [
+		'color-contrast',
+		'WCAG2AA.Principle2.Guideline2_4.2_4_1.G1,G123,G124.NoSuchID'
+	],
+	hideElements: '#bodyContent, #siteNotice',
+	chromeLaunchConfig: {
+		headless: true,
+		args: [
+			'--no-sandbox',
+			'--disable-setuid-sandbox'
+		]
+	}
+};
+
+module.exports = {
+	namespace: NAMESPACE,
+	paths: utils.makeA11yPaths( NAMESPACE ),
+	tests: [
+		{
+			name: 'default',
+			url: BASE_URL + '/wiki/Test',
+			...testDefaults
+		},
+		{
+			name: 'logged_in',
+			url: BASE_URL + '/wiki/Test',
+			...testDefaults
+		}
+	]
+};
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -75,3 +75,21 @@ services:
       - ./configCodex.js:/pixel/configCodex.js
       - ./src:/pixel/src
       - ./report:/pixel/report
+  a11y-regression:
+    init: true
+    network_mode: host
+    build:
+      context: .
+      dockerfile: Dockerfile.a11y-regression
+      args:
+        - PA11Y_VERSION=6.2.3
+    working_dir: /pixel
+    env_file:
+      - .env
+    volumes:
+      - ./context.json:/pixel/context.json
+      - ./viewports.js:/pixel/viewports.js
+      - ./utils.js:/pixel/utils.js
+      - ./configDesktopA11y.js:/pixel/configDesktopA11y.js
+      - ./src:/pixel/src
+      - ./report-a11y:/pixel/report-a11y
diff --git a/pixel.js b/pixel.js
@@ -92,27 +92,37 @@ ${markerString}`
 
 /**
  * @param {string} groupName
+ * @param {boolean} a11y
  * @return {string} path to configuration file.
  * @throws {Error} for unknown group
  */
-const getGroupConfig = ( groupName ) => {
-	switch ( groupName ) {
-		case 'web-maintained':
-			return 'configWebMaintained.js';
-		case 'echo':
-			return 'configEcho.js';
-		case 'desktop-dev':
-			return 'configDesktopDev.js';
-		case 'desktop':
-			return 'configDesktop.js';
-		case 'mobile':
-			return 'configMobile.js';
-		case 'campaign-events':
-			return 'configCampaignEvents.js';
-		case 'codex':
-			return 'configCodex.js';
-		default:
-			throw new Error( `Unknown test group: ${groupName}` );
+const getGroupConfig = ( groupName, a11y ) => {
+	if ( a11y ) {
+		switch ( groupName ) {
+			case 'desktop':
+				return 'configDesktopA11y.js';
+			default:
+				throw new Error( `Unknown test group: ${groupName}` );
+		}
+	} else {
+		switch ( groupName ) {
+			case 'web-maintained':
+				return 'configWebMaintained.js';
+			case 'echo':
+				return 'configEcho.js';
+			case 'desktop-dev':
+				return 'configDesktopDev.js';
+			case 'desktop':
+				return 'configDesktop.js';
+			case 'mobile':
+				return 'configMobile.js';
+			case 'campaign-events':
+				return 'configCampaignEvents.js';
+			case 'codex':
+				return 'configCodex.js';
+			default:
+				throw new Error( `Unknown test group: ${groupName}` );
+		}
 	}
 };
 
@@ -181,10 +191,9 @@ async function processCommand( type, opts ) {
 			context[ group ].description = description;
 		}
 		context[ group ][ type ] = active;
+
 		// store details of this run.
 		fs.writeFileSync( `${__dirname}/context.json`, JSON.stringify( context ) );
-		const configFile = getGroupConfig( group );
-		const config = require( `${__dirname}/${configFile}` );
 
 		// Start docker containers.
 		await batchSpawn.spawn(
@@ -200,45 +209,62 @@ async function processCommand( type, opts ) {
 			[ 'compose', ...getComposeOpts( [ 'exec', ...( process.env.NONINTERACTIVE ? [ '-T' ] : [] ), 'mediawiki', '/src/main.js', JSON.stringify( opts ) ] ) ]
 		);
 
-		// Remove test screenshots folder (if present) so that its size doesn't
-		// increase with each `test` run. BackstopJS automatically removes the
-		// reference folder when the `reference` command is run, but not the test
-		// folder when the `test` command is run.
-		if ( type === 'test' ) {
-			removeFolder( config.paths.bitmaps_test );
-		}
-		// Execute Visual regression tests.
-		await batchSpawn.spawn(
-			'docker',
-			[ 'compose', ...getComposeOpts( [ 'run', ...( process.env.NONINTERACTIVE ? [ '--no-TTY' ] : [] ), '--rm', 'visual-regression', type, '--config', configFile ] ) ]
-		).then( async () => {
-			await openReportIfNecessary(
-				type, group, config.paths.html_report, process.env.NONINTERACTIVE
-			);
-		}, async ( /** @type {Error} */ err ) => {
-			if ( err.message.includes( '130' ) ) {
-				// If user ends subprocess with a sigint, exit early.
-				// eslint-disable-next-line no-process-exit
-				process.exit( 1 );
-			}
+		const configFile = getGroupConfig( group, opts.a11y );
+		const config = require( `${__dirname}/${configFile}` );
+
+		if ( opts.a11y ) {
+			// Execute a11y regression tests.
+			await batchSpawn.spawn(
+				'docker',
+				[ 'compose', ...getComposeOpts( [ 'run', ...( process.env.NONINTERACTIVE ? [ '--no-TTY' ] : [] ), '--rm', 'a11y-regression', type, configFile ] ) ]
+			).finally( async () => {
+				// Reset the database if `--reset-db` option is passed.
+				if ( opts.resetDb ) {
+					console.log( 'Resetting database state...' );
+					await resetDb();
+				}
+			} );
+		} else {
+			// Execute Visual regression tests.
 
-			if ( err.message.includes( 'Exit with error code 1' ) ) {
+			// Remove test screenshots folder (if present) so that its size doesn't
+			// increase with each `test` run. BackstopJS automatically removes the
+			// reference folder when the `reference` command is run, but not the test
+			// folder when the `test` command is run.
+			if ( type === 'test' ) {
+				removeFolder( config.paths.bitmaps_test );
+			}
+			await batchSpawn.spawn(
+				'docker',
+				[ 'compose', ...getComposeOpts( [ 'run', ...( process.env.NONINTERACTIVE ? [ '--no-TTY' ] : [] ), '--rm', 'visual-regression', type, '--config', configFile ] ) ]
+			).then( async () => {
 				await openReportIfNecessary(
 					type, group, config.paths.html_report, process.env.NONINTERACTIVE
 				);
-				// eslint-disable-next-line no-process-exit
-				process.exit( 1 );
-			}
+			}, async ( /** @type {Error} */ err ) => {
+				if ( err.message.includes( '130' ) ) {
+					// If user ends subprocess with a sigint, exit early.
+					// eslint-disable-next-line no-process-exit
+					process.exit( 1 );
+				}
 
-			throw err;
-		} ).finally( async () => {
-			// Reset the database if `--reset-db` option is passed.
-			if ( opts.resetDb ) {
-				console.log( 'Resetting database state...' );
-				await resetDb();
-			}
-		} );
+				if ( err.message.includes( 'Exit with error code 1' ) ) {
+					await openReportIfNecessary(
+						type, group, config.paths.html_report, process.env.NONINTERACTIVE
+					);
+					// eslint-disable-next-line no-process-exit
+					process.exit( 1 );
+				}
 
+				throw err;
+			} ).finally( async () => {
+				// Reset the database if `--reset-db` option is passed.
+				if ( opts.resetDb ) {
+					console.log( 'Resetting database state...' );
+					await resetDb();
+				}
+			} );
+		}
 	} catch ( err ) {
 		console.error( err );
 		// eslint-disable-next-line no-process-exit
@@ -248,6 +274,10 @@ async function processCommand( type, opts ) {
 
 function setupCli() {
 	const { program } = require( 'commander' );
+	const a11yOpt = /** @type {const} */ ( [
+		'-a, --a11y',
+		'Run automated a11y tests in addition to visual regression.'
+	] );
 	const branchOpt = /** @type {const} */ ( [
 		'-b, --branch <name-of-branch>',
 		`Name of branch. Can be "${MAIN_BRANCH}" or a release branch (e.g. "origin/wmf/1.37.0-wmf.19"). Use "${LATEST_RELEASE_BRANCH}" to use the latest wmf release branch.`,
@@ -275,6 +305,7 @@ function setupCli() {
 		.command( 'reference' )
 		.description( 'Create reference (baseline) screenshots and delete the old reference screenshots.' )
 		.requiredOption( ...branchOpt )
+		.option( ...a11yOpt )
 		.option( ...changeIdOpt )
 		.option( ...groupOpt )
 		.option( ...resetDbOpt )
@@ -286,6 +317,7 @@ function setupCli() {
 		.command( 'test' )
 		.description( 'Create test screenshots and compare them against the reference screenshots.' )
 		.requiredOption( ...branchOpt )
+		.option( ...a11yOpt )
 		.option( ...changeIdOpt )
 		.option( ...groupOpt )
 		.option( ...resetDbOpt )

diff --git a/report-a11y/.gitignore b/report-a11y/.gitignore
@@ -0,0 +1,3 @@
+# Ignore everything except this file
+*
+!.gitignore