fix(reporter): Run inside isolated contexts (#3129)

This PR sets the axe reporters up so they can run without access to the top-level browsing context. This is needed so that `axe.finishRun()` can be called outside the context of a web page.

Co-authored-by: Steven Lambert <2433219+straker@users.noreply.github.com>
Co-authored-by: Stephen Mathieson <me@stephenmathieson.com>
Co-authored-by: Steven Lambert <2433219+straker@users.noreply.github.com>

.eslintrc.js

@@ -1,4 +1,5 @@
 module.exports = {
+  root: true,
   extends: ['prettier'],
   parserOptions: {
     ecmaVersion: 2021
@@ -70,6 +71,10 @@ module.exports = {
   overrides: [
     {
       files: ['lib/**/*.js'],
+      excludedFiles: [
+        'lib/core/reporters/**/*.js',
+        'lib/**/*-after.js'
+      ],
       parserOptions: {
         sourceType: 'module'
       },
@@ -87,6 +92,23 @@ module.exports = {
         'no-use-before-define': 'off'
       }
     },
+    {
+      // after functions and reporters will not be run inside the same context as axe.run so should not access browser globals that require context specific information (window.location, window.getComputedStyles, etc.)
+      files: [
+        'lib/**/*-after.js',
+        'lib/core/reporters/**/*.js'
+      ],
+      parserOptions: {
+        sourceType: 'module'
+      },
+      env: {},
+      globals: {},
+      rules: {
+        'func-names': [2, 'as-needed'],
+        'prefer-const': 2,
+        'no-use-before-define': 'off'
+      }
+    },
     {
       files: ['test/**/*.js'],
       parserOptions: {

axe.d.ts

@@ -96,13 +96,8 @@ declare namespace axe {
     preload?: boolean;
     performanceTimer?: boolean;
   }
-  interface AxeResults {
+  interface AxeResults extends EnvironmentData {
     toolOptions: RunOptions;
-    testEngine: TestEngine;
-    testRunner: TestRunner;
-    testEnvironment: TestEnvironment;
-    url: string;
-    timestamp: string;
     passes: Result[];
     violations: Result[];
     incomplete: Result[];
@@ -262,6 +257,7 @@ declare namespace axe {
   interface PartialResult {
     frames: SerialDqElement[];
     results: PartialRuleResult[];
+    environmentData?: EnvironmentData;
   }
   type PartialResults = Array<PartialResult | null>
   interface FrameContext {
@@ -272,6 +268,13 @@ declare namespace axe {
     getFrameContexts: (context?: ElementContext) => FrameContext[];
     shadowSelect: (selector: CrossTreeSelector) => Element | null;
   }
+  interface EnvironmentData {
+    testEngine: TestEngine;
+    testRunner: TestRunner;
+    testEnvironment: TestEnvironment;
+    url: string;
+    timestamp: string;
+  }
 
   let version: string;
   let plugins: any;

doc/run-partial.md

@@ -81,6 +81,13 @@ The `axe.utils.getFrameContexts` method takes any valid context, and returns an
 
 - `frameSelector`: This is a CSS selector, or array of CSS selectors in case of nodes in a shadow DOM tree to locate the frame element to be tested.
 - `frameContext`: This is an object is a context object that should be tested in the particular frame.
+
+## Custom Rulesets and Reporters
+
+Because `axe.finishRun` does not run inside the page, the `reporter` and `after` methods do not have access to the top-level `window` and `document` objects, and might not have access to common browser APIs. Axe-core reporter use the `environmentData` property that is set on the partialResult object of the initiator.
+
+Because of this constraint, custom reporters, and custom rulesets that add `after` methods must not rely on browser APIs or globals. Any data needed for either should either be taken from the `environmentData` property, or collected in an `evaluate` method of a check, and stored using its `.data()` method.
+
 ## Recommendations
 
 When building integrations with browser drivers using axe-core, it is safer and more stable to use `axe.runPartial` and `axe.finishRun` then to use `axe.run`. These two methods ensure that no information from one frame is ever handed off to another. That way if any script in a frame interferes with the `axe` object, or with `window.postMessage`, other frames will not be affected.

lib/core/public/finish-run.js

@@ -9,6 +9,7 @@ import {
 
 export default function finishRun(partialResults, options = {}) {
   options = clone(options);
+  const { environmentData } = partialResults.find(r => r.environmentData) || {}
 
   // normalize the runOnly option for the output of reporters toolOptions
   axe._audit.normalizeOptions(options);
@@ -20,7 +21,7 @@ export default function finishRun(partialResults, options = {}) {
   results.forEach(publishMetaData);
   results = results.map(finalizeRuleResult);
 
-  return createReport(results, options);
+  return createReport(results, { environmentData, ...options });
 }
 
 function setFrameSpec(partialResults) {

lib/core/public/run-partial.js

@@ -1,6 +1,6 @@
 import Context from '../base/context';
 import teardown from './teardown';
-import { DqElement, getSelectorData, assert } from '../utils';
+import { DqElement, getSelectorData, assert, getEnvironmentData } from '../utils';
 import normalizeRunParams from './run/normalize-run-params';
 
 export default function runPartial(...args) {
@@ -28,7 +28,12 @@ export default function runPartial(...args) {
       const frames = contextObj.frames.map(({ node }) => {
         return new DqElement(node, options).toJSON();
       });
-      return { results, frames };
+      let environmentData;
+      if (contextObj.initiator) {
+        environmentData = getEnvironmentData();
+      }
+
+      return { results, frames, environmentData };
     })
     .finally(() => {
       axe._running = false;

lib/core/public/run-virtual-rule.js

@@ -5,6 +5,7 @@ import {
   publishMetaData,
   finalizeRuleResult,
   aggregateResult,
+  getEnvironmentData,
   getRule
 } from '../utils';
 
@@ -54,7 +55,7 @@ function runVirtualRule(ruleId, vNode, options = {}) {
   );
 
   return {
-    ...helpers.getEnvironmentData(),
+    ...getEnvironmentData(),
     ...results,
     toolOptions: options
   };

lib/core/reporters/helpers/index.js

@@ -1,5 +1,4 @@
 import failureSummary from './failure-summary';
-import getEnvironmentData from './get-environment-data';
 import incompleteFallbackMessage from './incomplete-fallback-msg';
 import processAggregate from './process-aggregate';
 
@@ -8,14 +7,12 @@ import processAggregate from './process-aggregate';
 axe._thisWillBeDeletedDoNotUse = axe._thisWillBeDeletedDoNotUse || {};
 axe._thisWillBeDeletedDoNotUse.helpers = {
   failureSummary,
-  getEnvironmentData,
   incompleteFallbackMessage,
   processAggregate
 };
 
 export {
   failureSummary,
-  getEnvironmentData,
   incompleteFallbackMessage,
   processAggregate
 };

lib/core/reporters/na.js

@@ -1,23 +1,20 @@
-import { processAggregate, getEnvironmentData } from './helpers';
+import { processAggregate } from './helpers';
+import { getEnvironmentData } from '../utils';
 
 const naReporter = (results, options, callback) => {
   console.warn(
     '"na" reporter will be deprecated in axe v4.0. Use the "v2" reporter instead.'
   );
-
   if (typeof options === 'function') {
     callback = options;
     options = {};
   }
 
-  var out = processAggregate(results, options);
+  const { environmentData, ...toolOptions } = options;
   callback({
-    ...getEnvironmentData(),
-    toolOptions: options,
-    violations: out.violations,
-    passes: out.passes,
-    incomplete: out.incomplete,
-    inapplicable: out.inapplicable
+    ...getEnvironmentData(environmentData),
+    toolOptions,
+    ...processAggregate(results, options)
   });
 };
 

lib/core/reporters/no-passes.js

@@ -1,19 +1,21 @@
-import { processAggregate, getEnvironmentData } from './helpers';
+import { processAggregate } from './helpers';
+import { getEnvironmentData } from '../utils';
 
 const noPassesReporter = (results, options, callback) => {
   if (typeof options === 'function') {
     callback = options;
     options = {};
   }
+  const { environmentData, ...toolOptions } = options;
   // limit result processing to types we want to include in the output
   options.resultTypes = ['violations'];
 
-  var out = processAggregate(results, options);
+  var { violations } = processAggregate(results, options);
 
   callback({
-    ...getEnvironmentData(),
-    toolOptions: options,
-    violations: out.violations
+    ...getEnvironmentData(environmentData),
+    toolOptions,
+    violations
   });
 };
 

lib/core/reporters/raw-env.js

@@ -1,17 +1,16 @@
-import { getEnvironmentData } from './helpers';
+import { getEnvironmentData } from '../utils';
 import rawReporter from './raw';
 
 const rawEnvReporter = (results, options, callback) => {
   if (typeof options === 'function') {
     callback = options;
     options = {};
   }
-  function rawCallback(raw) {
-    const env = getEnvironmentData();
+  const { environmentData, ...toolOptions } = options;
+  rawReporter(results, toolOptions, (raw) => {
+    const env = getEnvironmentData(environmentData);
     callback({ raw, env });
-  }
-
-  rawReporter(results, options, rawCallback);
+  });
 };
 
 export default rawEnvReporter;

lib/core/reporters/v1.js

@@ -1,15 +1,13 @@
-import {
-  processAggregate,
-  failureSummary,
-  getEnvironmentData
-} from './helpers';
+import { processAggregate, failureSummary } from './helpers';
+import { getEnvironmentData } from '../utils'
 
 const v1Reporter = (results, options, callback) => {
   if (typeof options === 'function') {
     callback = options;
     options = {};
-  }
-  var out = processAggregate(results, options);
+  };
+  const { environmentData, ...toolOptions } = options;
+  const out = processAggregate(results, options);
 
   const addFailureSummaries = result => {
     result.nodes.forEach(nodeResult => {
@@ -21,12 +19,9 @@ const v1Reporter = (results, options, callback) => {
   out.violations.forEach(addFailureSummaries);
 
   callback({
-    ...getEnvironmentData(),
-    toolOptions: options,
-    violations: out.violations,
-    passes: out.passes,
-    incomplete: out.incomplete,
-    inapplicable: out.inapplicable
+    ...getEnvironmentData(environmentData),
+    toolOptions,
+    ...out
   });
 };
 

lib/core/reporters/v2.js

@@ -1,18 +1,17 @@
-import { processAggregate, getEnvironmentData } from './helpers';
+import { processAggregate } from './helpers';
+import { getEnvironmentData } from '../utils';
 
 const v2Reporter = (results, options, callback) => {
   if (typeof options === 'function') {
     callback = options;
     options = {};
   }
+  const { environmentData, ...toolOptions } = options;
   var out = processAggregate(results, options);
   callback({
-    ...getEnvironmentData(),
-    toolOptions: options,
-    violations: out.violations,
-    passes: out.passes,
-    incomplete: out.incomplete,
-    inapplicable: out.inapplicable
+    ...getEnvironmentData(environmentData),
+    toolOptions,
+    ...out
   });
 };
 

lib/core/utils/get-environment-data.js

@@ -0,0 +1,47 @@
+/**
+ * Add information about the environment axe was run in.
+ * @return {EnvironmentData}
+ */
+export default function getEnvironmentData(metadata = null, win = window) {
+  if (metadata && typeof metadata === 'object') {
+    return metadata;
+  } else if (typeof win !== 'object') {
+    return {}
+  }
+
+  return {
+    testEngine: {
+      name: 'axe-core',
+      version: axe.version
+    },
+    testRunner: {
+      name: axe._audit.brand
+    },
+    testEnvironment: getTestEnvironment(win),
+    timestamp: new Date().toISOString(),
+    url: win.location?.href
+  };
+}
+
+function getTestEnvironment(win) {
+  if (!win.navigator || typeof win.navigator !== 'object') {
+    return {}
+  }
+  const { navigator, innerHeight, innerWidth } = win;
+  const { angle, type } = getOrientation(win) || {}
+  return {
+    userAgent: navigator.userAgent,
+    windowWidth: innerWidth,
+    windowHeight: innerHeight,
+    orientationAngle: angle,
+    orientationType: type
+  }
+}
+
+function getOrientation({ screen }) {
+  return (
+    screen.orientation ||
+    screen.msOrientation ||
+    screen.mozOrientation
+  );
+}