Add ability to filter scenarios and watch Backstop runs as they execute

README.md

@@ -251,6 +251,25 @@ repo.
 
 Please file all bugs, requests, and issues on the [web team's visual regression phabricator board](https://phabricator.wikimedia.org/project/board/5933/)
 
+## Debugging
+
+The following env vars are useful when running and debugging Pixel locally
+
+### Watch mode
+
+To watch BackstopJS taking your screenshots (in its visual-regression container) you can use the `WATCH_MODE` env var:
+
+`WATCH_MODE=1 ./pixel.js reference`
+
+This can be very helpful determining Mediawiki related configuration issues and should work with any command kicking off Pixel BackstopJS runs
+
+### Filter
+
+To quickly restrict which scenario runs without modifying any code, you can use the `FILTER` env var:
+
+`FILTER="Sticky header with unpinned TOC" ./pixel.js reference`
+
+This is useful when debugging specific scenarios and should work with any command kicking off Pixel BackstopJS runs
 
 ## Roadmap
 

config/configCommon.js

@@ -1,4 +1,4 @@
-module.exports = {
+const config = {
 	id: 'MediaWiki',
 	asyncCaptureLimit: 4,
 	asyncCompareLimit: 25,
@@ -17,3 +17,37 @@ module.exports = {
 	debugWindow: false,
 	delay: 100
 };
+
+const watchConfig = {
+  asyncCaptureLimit: 1,
+  engineOptions: {
+    headless: false,
+    slowMo: 100,
+  },
+  delay: 1000
+};
+
+const deepMerge = (target, source) => {
+  Object.keys(source).forEach(key => {
+    const sourceValue = source[key];
+    if (Array.isArray(target[key]) && Array.isArray(sourceValue)) {
+      target[key] = [...new Set([...target[key], ...sourceValue])];
+    } else if (sourceValue && typeof sourceValue === 'object') {
+      target[key] = deepMerge(target[key] ?? {}, sourceValue);
+    } else {
+      target[key] = sourceValue;
+    }
+  });
+  return target;
+};
+
+if ( process.env.WATCH_MODE === '1' ) {
+  deepMerge(config, watchConfig);
+}
+
+console.log('-----------------------------');
+console.log('Current common configuration:');
+console.log(JSON.stringify(config, null, 2));
+console.log('-----------------------------');
+
+module.exports = config;

docker-compose.yml

@@ -63,7 +63,20 @@ services:
       retries: 20
     networks:
       - pixel_network
+  novnc:
+    profiles:
+      - manual
+    build: novnc
+    environment:
+      - DISPLAY_WIDTH=1280
+      - DISPLAY_HEIGHT=1024
+    networks:
+      - pixel_network
+    ports:
+        - 8088:8088
   visual-regression:
+    profiles:
+      - manual
     init: true
     build:
       context: .
@@ -73,6 +86,8 @@ services:
     working_dir: /pixel
     env_file:
       - .env
+    environment:
+      - DISPLAY=novnc:0
     volumes:
       - ./context.json:/pixel/context.json
       - ./viewports.js:/pixel/viewports.js
@@ -86,6 +101,8 @@ services:
     networks:
       - pixel_network
   a11y-regression:
+    profiles:
+      - manual
     init: true
     build:
       context: .
@@ -95,6 +112,8 @@ services:
     working_dir: /pixel
     env_file:
       - .env
+    environment:
+      - DISPLAY=novnc:0
     volumes:
       - ./context.json:/pixel/context.json
       - ./viewports.js:/pixel/viewports.js

novnc/Dockerfile

@@ -0,0 +1,24 @@
+FROM alpine
+
+RUN set -ex; \
+    apk add --no-cache \
+        xfce4 \
+        novnc \
+        supervisor \
+        x11vnc \
+        xvfb \
+    && rm -rf /var/cache/apk/*
+
+ENV HOME=/root \
+    LANG=en_US.UTF-8 \
+    LANGUAGE=en_US.UTF-8 \
+    LC_ALL=C.UTF-8 \
+    DISPLAY=:0.0 \
+    DISPLAY_WIDTH=1280 \
+    DISPLAY_HEIGHT=1024
+
+COPY . /app
+COPY background.svg /usr/share/backgrounds/xfce/xfce-shapes.svg
+
+CMD ["/app/entrypoint.sh"]
+EXPOSE 8088

novnc/conf.d/desktop.conf

@@ -0,0 +1,3 @@
+[program:xfce4]
+command=startxfce4
+autorestart=true

novnc/conf.d/websockify.conf

@@ -0,0 +1,3 @@
+[program:websockify]
+command=websockify --web /usr/share/novnc 8088 localhost:5900
+autorestart=true

novnc/conf.d/x11vnc.conf

@@ -0,0 +1,3 @@
+[program:x11vnc]
+command=x11vnc -forever -shared
+autorestart=true

novnc/conf.d/xvfb.conf

@@ -0,0 +1,3 @@
+[program:xvfb]
+command=Xvfb :0 -screen 0 "%(ENV_DISPLAY_WIDTH)s"x"%(ENV_DISPLAY_HEIGHT)s"x24 -listen tcp -ac
+autorestart=true

novnc/ensure-novnc-container-up.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+if docker container ls --format '{{.Names}}' | grep -q "pixel-novnc-1"; then
+  echo "The pixel-novnc-1 container is already running."
+else
+  docker compose up --no-deps --detach novnc
+fi

novnc/entrypoint.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+set -eux
+
+exec supervisord -c /app/supervisord.conf

novnc/open-watch-url.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+URL="http://localhost:8088/vnc_lite.html?autoconnect=true"
+
+open "$URL" 2>/dev/null || {
+    printf "\n\033[0;32mThe 'open' command is not available\033[0m\n"
+    printf "\n\033[0;33mOpen the following URL in your web browser to watch BackstopJS do screen captures inside its docker container:\033[0m\n"
+    printf "\n\033[1;33m%s\033[0m\n\n" "$URL"
+    sleep 5
+}

novnc/supervisord.conf

@@ -0,0 +1,5 @@
+[supervisord]
+nodaemon=true
+
+[include]
+files = /app/conf.d/*.conf

pixel.js

@@ -165,6 +165,10 @@ async function processCommand( type, opts, runSilently = false ) {
 		const { stdout: stdout2 } = await simpleSpawn.exec( './reset-db.sh' );
 		console.log( stdout2 );
 
+		if (process.env.WATCH_MODE === '1') {
+			await simpleSpawn.spawn( './novnc/open-watch-url.sh' );
+		}
+
 		if ( opts.a11y ) {
 			return await runA11yRegressionTests( type, configFile, opts.logResults, opts );
 		} else {
@@ -266,8 +270,21 @@ async function runVisualRegressionTests( type, config, group, runSilently, confi
 	writeRunInProgressTemplateToIndexFile( indexFileFullPath, group );
 
 	const finished = await simpleSpawn.spawn(
-		'docker',
-		[ 'compose', ...getComposeOpts( [ 'run', ...( process.env.NONINTERACTIVE ? [ '--no-TTY' ] : [] ), '--rm', 'visual-regression', type, '--config', configFile ] ) ]
+	  'docker',
+	  [
+	    'compose',
+	    ...getComposeOpts( [
+	      'run',
+	      ...( process.env.NONINTERACTIVE ? [ '--no-TTY' ] : [] ),
+	      '--rm',
+	      ...( process.env.WATCH_MODE ? [ '-e', `WATCH_MODE=${process.env.WATCH_MODE}` ] : [] ),
+	      'visual-regression',
+	      type,
+	      '--config',
+	      configFile,
+	      ...( process.env.FILTER ? [ '--filter', `${process.env.FILTER}` ] : [] )
+	    ] )
+	  ]
 	).then( async () => {
 		if ( type !== 'reference' ) {
 			await addBannerAndIfNecessaryOpenReport(

start.sh

@@ -1,4 +1,9 @@
 #!/bin/bash
 
 # Start docker containers
-docker compose --progress=plain --project-directory . -f ./docker-compose.yml up -d
+docker compose --progress=plain --project-directory . -f ./docker-compose.yml up -d
+
+# Start novnc container if needed
+if [ "${WATCH_MODE}" = "1" ]; then
+  ./novnc/ensure-novnc-container-up.sh
+fi