[RFC][Development workflow] Pa11y performance bump

.nvmrc

@@ -1 +1 @@
-v10.15.0
+v10.18.1

.travis.yml

@@ -1,6 +1,6 @@
 language: node_js
 node_js:
-  - '10.15.0'
+  - '10.18.1'
 cache: yarn
 
 jobs:

UNRELEASED.md

@@ -14,6 +14,8 @@ Use [the changelog guidelines](https://git.io/polaris-changelog-guidelines) to f
 
 ### Development workflow
 
+- Update pa11y test to focus on kitchen sink pages ([#3280](https://github.com/Shopify/polaris-react/pull/3280))
+
 ### Dependency upgrades
 
 ### Code quality

dev.yml

@@ -2,7 +2,7 @@ name: polaris-react
 up:
   - node:
       yarn: v1.13.0
-      version: v10.15.0 # to be kept in sync with .nvmrc and .travis.yml
+      version: v10.18.1 # to be kept in sync with .nvmrc and .travis.yml
   - git_hooks:
       pre-commit: pre-commit
 

package.json

@@ -124,7 +124,7 @@
     "babel-core": "7.0.0-bridge.0",
     "babel-loader": "^8.1.0",
     "babel-preset-shopify": "^21.0.0",
-    "chalk": "^2.4.2",
+    "chalk": "^4.1.0",
     "change-case": "^3.1.0",
     "chromatic": "^5.0.0",
     "codecov": "^3.5.0",
@@ -146,10 +146,10 @@
     "node-sass": "^4.12.0",
     "npm-run-all": "^4.1.5",
     "object-hash": "^1.3.1",
-    "pa11y": "^5.2.0",
+    "pa11y": "^5.3.0",
     "postcss": "^7.0.18",
     "postcss-modules": "^3.1.0",
-    "puppeteer": "^1.20.0",
+    "puppeteer": "^5.3.0",
     "react": "^16.9.0",
     "react-dom": "^16.9.0",
     "react-is": "^16.9.0",

scripts/pa11y.js

@@ -1,151 +1,65 @@
 /* eslint-disable no-console */
 const puppeteer = require('puppeteer');
 const pa11y = require('pa11y');
+const chalk = require('chalk');
+
+// eslint-disable-next-line node/no-path-concat
+const iframePath = `file://${__dirname}/../build/storybook/static/iframe.html`;
+const sinkUrl = `${iframePath}?id=playground-playground--kitchen-sink&viewMode=story`;
+
+const testUrls = [
+  sinkUrl,
+  `${sinkUrl}&contexts=Global%20Theming=Enabled%20-%20Light%20Mode`,
+  // Dark mode has lots of errors. It is still very WIP so ignore for now
+  // `${sinkUrl}&contexts=Global%20Theming=Enabled%20-%20Dark%20Mode`,
+];
+
+const printTitle = ({type, message}) => {
+  switch (type) {
+    case 'error':
+      return `${chalk.red('Error:')} ${message}`;
+    case 'warning':
+      return `${chalk.yellow('Warning:')} ${message}`;
+    case 'notice':
+      return `${chalk.blue('Notice:')} ${message}`;
+  }
+};
 
-const NUMBER_OF_BROWSERS = 5;
-
-async function runPa11y() {
-  const browsers = [
-    {
-      browser: await puppeteer.launch(),
-      taken: new Promise((resolve) => {
-        resolve();
-      }),
-    },
-    {
-      browser: await puppeteer.launch(),
-      taken: new Promise((resolve) => {
-        resolve();
-      }),
-    },
-    {
-      browser: await puppeteer.launch(),
-      taken: new Promise((resolve) => {
-        resolve();
-      }),
-    },
-    {
-      browser: await puppeteer.launch(),
-      taken: new Promise((resolve) => {
-        resolve();
-      }),
-    },
-    {
-      browser: await puppeteer.launch(),
-      taken: new Promise((resolve) => {
-        resolve();
-      }),
-    },
-  ];
-
-  await browsers.forEach(async (instance) => {
-    // eslint-disable-next-line require-atomic-updates
-    instance.page = await instance.browser.newPage();
-  });
-
-  let browserIndex = 0;
-
-  const results = [];
-
-  const setupBrowser = browsers[0].browser;
-  const page = await setupBrowser.newPage();
-
-  // eslint-disable-next-line node/no-path-concat
-  const iframePath = `file://${__dirname}/../build/storybook/static/iframe.html`;
+const allowedErrors = [
+  'Duplicate id attribute value "AppFrameMain" found on the web page.',
+  'Duplicate id attribute value "AppFrameMainContent" found on the web page.',
+];
 
-  const stories = await page
-    .goto(iframePath)
-    .then(() => page.evaluate(() => window.__STORYBOOK_CLIENT_API__.raw()));
+const testPages = async (urls) => {
+  const browser = await puppeteer.launch();
 
-  const storyQueryStrings = stories.reduce((memo, story) => {
-    // There is no need to test the Playground, or the "All Examples" stories
-    const isSkippedStory =
-      story.kind === 'Playground/Playground' || story.name === 'All Examples';
+  const testPage = async (url) => {
+    const page = await browser.newPage();
+    const result = await pa11y(url, {browser});
+    await page.close();
+    return result;
+  };
 
-    if (!isSkippedStory) {
-      const idParam = `id=${encodeURIComponent(story.id)}`;
-      memo.push(
-        idParam,
-        `${idParam}&contexts=Global%20Theming=Enabled%20-%20Light%20Mode`,
-        // Dark mode has lots of errors. It is still very WIP so ignore for now
-        // `${idParam}&contexts=Global%20Theming=Enabled%20-%20Dark%20Mode`,
-      );
-    }
-    return memo;
-  }, []);
+  const pagePromises = urls.map((url) => testPage(url));
+  const pageResults = await Promise.all(pagePromises);
 
-  storyQueryStrings.forEach((queryString) => {
-    const currentBrowser = browsers[browserIndex % NUMBER_OF_BROWSERS];
-    browserIndex++;
-    currentBrowser.taken = currentBrowser.taken.then(async () => {
-      console.log('Testing ', queryString);
-      const result = await pa11y(`${iframePath}?${queryString}`, {
-        browser: currentBrowser.browser,
-        ignore: [
-          // Missing lang attribute on <html> tag
-          // Storybook does not include this property so ignore it
-          'WCAG2AA.Principle3.Guideline3_1.3_1_1.H57.2',
-        ],
-      });
-      result.exampleID = queryString;
-      delete result.pageUrl;
-      results.push(result);
-    });
-  });
+  await browser.close();
 
-  await Promise.all(browsers.map((instance) => instance.taken));
-  await Promise.all(browsers.map((instance) => instance.browser.close()));
-  return results;
-}
+  const issues = [].concat(...pageResults.map((result) => result.issues));
+  return issues
+    .filter((issue) => !allowedErrors.includes(issue.message))
+    .map((issue) => `${chalk.bold(printTitle(issue))}\n${issue.context}`);
+};
 
 (async () => {
-  let rawResults;
-  try {
-    rawResults = await runPa11y();
-
-    if (rawResults.length === 0) {
-      throw new Error('Component URLs could not be crawled');
-    }
-  } catch (error) {
-    console.log(error);
-    process.exit(1);
-  }
-
-  const results = rawResults.filter((result) => result.issues.length);
-
-  if (results.length) {
-    console.log(
-      `
-
-========================================================================
-The following issues were discovered and need to be fixed before this code can be merged
-========================================================================
-`,
-    );
-
-    results.forEach((result) => {
-      console.log(
-        '------------------------------------------------------------------------',
-      );
-      console.log(result.exampleID);
-      console.log(
-        '------------------------------------------------------------------------',
-      );
-      console.log(JSON.stringify(result.issues, null, 2));
-    });
-  } else {
-    console.log(
-      `
-
-========================================================================
-No issues were discovered!
-========================================================================
-`,
-    );
-  }
+  const results = await testPages(testUrls);
 
   if (results.length) {
+    console.log(results.join('\n\n'));
+    console.log(chalk.bold(`\nFound ${results.length} issues testing URLs:`));
+    console.log(`- ${testUrls.join('\n -')}`);
     process.exit(1);
   }
+  console.log(`${chalk.bold.green('Success: ')} No issues were discovered!`);
   process.exit(0);
 })();

src/components/AppProvider/README.md

@@ -60,8 +60,8 @@ AppProvider works by default without any additional options passed to it.
         showHeader
         items={[
           {
-            id: 341,
-            url: 'customers/341',
+            id: 340,
+            url: 'customers/340',
             name: 'Mae Jemison',
             location: 'Decatur, USA',
           },
@@ -126,8 +126,8 @@ With an `i18n`, `AppProvider` will provide these translations to polaris compone
             location: 'Decatur, USA',
           },
           {
-            id: 256,
-            url: 'customers/256',
+            id: 255,
+            url: 'customers/255',
             name: 'Ellen Ochoa',
             location: 'Los Angeles, USA',
           },