diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index b412337c0..41f96dc63 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -36,7 +36,7 @@ jobs: - name: Check that there are no unexpected Sphinx warnings if: matrix.python-version == '3.10' - run: python tests/check_warnings.py + run: python tests/utils/check_warnings.py - name: Run the tests run: | diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d3b8efdf4..29297b792 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -56,6 +56,9 @@ jobs: # windows test - os: windows-latest python-version: "3.11" + # needed to cache the browsers for the accessibility tests + env: + PLAYWRIGHT_BROWSERS_PATH: ${{ github.workspace }}/pw-browsers runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 @@ -90,7 +93,7 @@ jobs: pip install nox nox -s compile - name: Run tests - run: pytest --color=yes --cov pydata_sphinx_theme --cov-branch --cov-report xml:cov.xml --cov-report= --cov-fail-under ${{ env.COVERAGE_THRESHOLD }} + run: pytest -m "not a11y" --color=yes --cov pydata_sphinx_theme --cov-branch --cov-report xml:cov.xml --cov-report= --cov-fail-under ${{ env.COVERAGE_THRESHOLD }} - name: Upload to Codecov if: matrix.python-version == '3.11' && matrix.os == 'ubuntu-latest' uses: codecov/codecov-action@v3.1.2 @@ -98,6 +101,22 @@ jobs: files: cov.xml fail_ci_if_error: true + # note I am setting this on top of the Python cache as I could not find + # how to set the hash key on the python one + - name: Set up browser cache (for accessibility tests) + uses: actions/cache@v3 + with: + path: | + ${{ github.workspace }}/pw-browsers + key: ${{ runner.os }}-pw-${{ hashFiles('pyproject.toml') }} + restore-keys: | + ${{ runner.os }}-pw- + + - name: Run accessibility tests with playwright + if: matrix.python-version == '3.11' && matrix.os == 'ubuntu-latest' + run: | + nox -s a11y + # Build our site on the 3 major OSes and check for Sphinx warnings build-site: needs: [lint] @@ -124,7 +143,7 @@ jobs: - name: Build docs run: sphinx-build -b html docs/ docs/_build/html --keep-going -w warnings.txt - name: Check for unexpected Sphinx warnings - run: python tests/check_warnings.py + run: python tests/utils/check_warnings.py # Run local Lighthouse audit against built site audit: diff --git a/docs/community/setup.md b/docs/community/setup.md index 3e732e14f..6777a53cd 100644 --- a/docs/community/setup.md +++ b/docs/community/setup.md @@ -22,10 +22,10 @@ The sections below cover the steps to do this in more detail. ## Clone the repository -First off you'll need your own copy of the `pydata-sphinx-theme` codebase. +First off you'll need your copy of the `pydata-sphinx-theme` codebase. You can clone it for local development like so: -1. **Fork the repository** so you have your own copy on GitHub. +1. **Fork the repository**, so you have your own copy on GitHub. See [the GitHub forking guide](https://docs.github.com/en/get-started/quickstart/fork-a-repo) for more information. 2. **Clone the repository locally** so that you have a local copy to work from: @@ -36,14 +36,14 @@ You can clone it for local development like so: ## Install your tools -Building a Sphinx site uses a combination of Python and Jinja to manage HTML, SCSS, and Javascript. +Building a Sphinx site uses a combination of Python and Jinja to manage HTML, SCSS, and JavaScript. To simplify this process, we use a few helper tools: -- [The Sphinx Theme Builder](https://sphinx-theme-builder.readthedocs.io/en/latest/) to automatically perform compilation of web assets. +- [The Sphinx Theme Builder](https://sphinx-theme-builder.readthedocs.io/en/latest/) compiles web assets in an automated way. - [pre-commit](https://pre-commit.com/) for automatically enforcing code standards and quality checks before commits. -- [nox](https://nox.thea.codes/), for automating common development tasks. +- [nox](https://nox.thea.codes/) for automating common development tasks. -In particular, `nox` can be used to automatically create isolated local development environments with all of the correct packages installed to work on the theme. +In particular, `nox` can be used to automatically create isolated local development environments with all the correct packages installed to work on the theme. The rest of this guide focuses on using `nox` to start with a basic environment. ```{seealso} @@ -58,9 +58,9 @@ To start, install `nox`: $ pip install nox ``` -You can call `nox` from the command line in order to perform common actions that are needed in building the theme. +You can call `nox` from the command line to perform common actions that are needed in building the theme. `nox` operates with isolated environments, so each action has its own packages installed in a local directory (`.nox`). -For common development actions, you'll simply need to use `nox` and won't need to set up any other packages. +For common development actions, you'll only need to use `nox` and won't need to set up any other packages. ### Setup `pre-commit` @@ -116,7 +116,7 @@ Now that you've built the documentation, edit one of the source files to see how 1. **Make an edit to a page**. For example, add a word or fix a typo on any page. 2. **Rebuild the documentation** with `nox -s docs` -It should go much faster this time, because `nox` is re-using the old environment, and because Sphinx has cached the pages that you didn't change. +It should go much faster this time because `nox` is re-using the previously created environment, and because Sphinx has cached the pages that you didn't change. ## Compile the CSS/JS assets @@ -138,7 +138,7 @@ The `sphinx-theme-builder` will bundle these assets automatically when we make a ## Run a development server -You can combine the above two actions and run a development server so that changes to `src/` are automatically bundled with the package, and the documentation is immediately reloaded in a live preview window. +You can combine the above two actions (build the docs and compile JS/CSS assets) and run a development server so that changes to `src/` are automatically bundled with the package, and the documentation is immediately reloaded in a live preview window. To run the development server with `nox`, run the following command: @@ -146,7 +146,7 @@ To run the development server with `nox`, run the following command: $ nox -s docs-live ``` -When working on the theme, saving changes to any of these directories: +When working on the theme, making changes to any of these directories: - `src/js/index.js` - `src/scss/index.scss` @@ -161,19 +161,33 @@ will cause the development server to do the following: ## Run the tests -This theme uses `pytest` for its testing, with a lightweight fixture defined -in the `test_build.py` script that makes it easy to run a Sphinx build using -this theme and inspect the results. +This theme uses `pytest` for its testing. There is a lightweight fixture defined +in the `test_build.py` script that makes it straightforward to run a Sphinx build using +this theme and inspect the results. There are also several automated accessibility checks in +`test_a11y.py`. -In addition, we use [pytest-regressions](https://pytest-regressions.readthedocs.io/en/latest/) -to ensure that the HTML generated by the theme is what we'd expect. This module +```{warning} +Currently, the automated accessibility tests check the Kitchen Sink page only. +We are working on extending coverage to the rest of the theme. +``` + +In addition, we use +[pytest-regressions](https://pytest-regressions.readthedocs.io/en/latest/) to +ensure that the HTML generated by the theme is what we'd expect. This module provides a `file_regression` fixture that will check the contents of an object against a reference file on disk. If the structure of the two differs, then the test will fail. If we _expect_ the structure to differ, then delete the file on -disk and run the test. A new file will be created, and subsequent tests will pass. +disk and run the test. A new file will be created, and subsequent tests will +pass. -To run the tests with `nox`, run the following command: +To run the build tests with `nox`, run the following command: ```console $ nox -s test ``` + +To run the accessibility checks: + +```console +$ nox -s a11y +``` diff --git a/docs/community/topics/accessibility.md b/docs/community/topics/accessibility.md index 5bff788de..b667a195e 100644 --- a/docs/community/topics/accessibility.md +++ b/docs/community/topics/accessibility.md @@ -1,11 +1,18 @@ # Accessibility checks -The accessibility checking tools can find a number of common HTML patterns which +```{note} +April-2023: we are currently +[re-evaluating how we do accessibility checks](https://github.com/pydata/pydata-sphinx-theme/issues/1168) +and reporting, so this may change soon. +``` + +In general, accessibility-checking tools can find a limited number of common HTML patterns which assistive technology can't help users understand. -We run a [Lighthouse](https://developers.google.com/web/tools/lighthouse) job in our CI/CD, which generates a "score" for all pages in our **Kitchen Sink** example documentation. -The configuration for Lighthouse is in: -- `.github/workflows/lighthouserc.json` +## Accessibility checks as part of our development process + +We run a [Lighthouse](https://developers.google.com/web/tools/lighthouse) job in our CI/CD, which generates a "score" for all pages in our **Kitchen Sink** example documentation. +The configuration for Lighthouse can be found in the `.github/workflows/lighthouserc.json` file. -For more information about configuring lighthouse, see [the lighthouse documentation](https://github.com/GoogleChrome/lighthouse-ci/blob/main/docs/configuration.md). +For more information about configuring Lighthouse, see [the Lighthouse documentation](https://github.com/GoogleChrome/lighthouse-ci/blob/main/docs/configuration.md). For more information about Accessibility in general, see [](../../user_guide/accessibility.rst). diff --git a/noxfile.py b/noxfile.py index 71903094f..24fe688f3 100644 --- a/noxfile.py +++ b/noxfile.py @@ -5,6 +5,7 @@ nox -s docs -- -r """ +import os import shutil as sh import tempfile from pathlib import Path @@ -87,7 +88,28 @@ def test(session: nox.Session) -> None: if _should_install(session): session.install("-e", ".[test]") session.run(*split("pybabel compile -d src/pydata_sphinx_theme/locale -D sphinx")) - session.run("pytest", *session.posargs) + session.run("pytest", "-m", "not a11y", *session.posargs) + + +@nox.session() +def a11y(session: nox.Session) -> None: + """Run the accessibility test suite only.""" + if _should_install(session): + session.install("-e", ".[test, a11y]") + # Install the drivers that Playwright needs to control the browsers. + if os.environ.get("CI") or os.environ.get("GITPOD_WORKSPACE_ID"): + # CI and other cloud environments are potentially missing system + # dependencies, so we tell Playwright to also install the system + # dependencies + session.run("playwright", "install", "--with-deps") + else: + # But most dev environments have the needed system dependencies + session.run("playwright", "install") + # Build the docs so we can run accessibility tests against them. + session.run("nox", "-s", "docs") + # The next step would be to open a server to the docs for Playwright, but + # that is done in the test file, along with the accessibility checks. + session.run("pytest", "-m", "a11y", *session.posargs) @nox.session(name="test-sphinx") diff --git a/package-lock.json b/package-lock.json index bfb1c6ef7..89c5c3906 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,6 +11,7 @@ "bootstrap": "^5.2.2" }, "devDependencies": { + "axe-core": "^4.6.3", "copy-webpack-plugin": "^11.0.0", "css-loader": "^3.4.2", "css-minimizer-webpack-plugin": "^4.2.2", @@ -312,9 +313,9 @@ } }, "node_modules/@types/yargs": { - "version": "17.0.23", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.23.tgz", - "integrity": "sha512-yuogunc04OnzGQCrfHx+Kk883Q4X0aSwmYZhKjI21m+SVYzjIbrWl8dOOwSv5hf2Um2pdCOXWo9isteZTNXUZQ==", + "version": "17.0.24", + "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.24.tgz", + "integrity": "sha512-6i0aC7jV6QzQB8ne1joVZ0eSFIstHsCrobmOtghM11yGlH0j43FKL2UhWdELkyps0zuf7qVTUVCCR+tgSlyLLw==", "dev": true, "dependencies": { "@types/yargs-parser": "*" @@ -675,6 +676,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/axe-core": { + "version": "4.6.3", + "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.6.3.tgz", + "integrity": "sha512-/BQzOX780JhsxDnPpH4ZiyrJAzcd8AfzFPkv+89veFSr1rcMjuq2JDCwypKaPeB6ljHp9KjXhPpjgCvQlWYuqg==", + "dev": true, + "engines": { + "node": ">=4" + } + }, "node_modules/big.js": { "version": "5.2.2", "resolved": "https://registry.npmjs.org/big.js/-/big.js-5.2.2.tgz", @@ -808,9 +818,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001470", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001470.tgz", - "integrity": "sha512-065uNwY6QtHCBOExzbV6m236DDhYCCtPmQUCoQtwkVqzud8v5QPidoMr6CoMkC2nfp6nksjttqWQRRh75LqUmA==", + "version": "1.0.30001472", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001472.tgz", + "integrity": "sha512-xWC/0+hHHQgj3/vrKYY0AAzeIUgr7L9wlELIcAvZdDUHlhL/kNxMdnQLOSOQfP8R51ZzPhmHdyMkI0MMpmxCfg==", "dev": true, "funding": [ { @@ -820,6 +830,10 @@ { "type": "tidelift", "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" } ] }, @@ -1854,9 +1868,9 @@ } }, "node_modules/electron-to-chromium": { - "version": "1.4.340", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.340.tgz", - "integrity": "sha512-zx8hqumOqltKsv/MF50yvdAlPF9S/4PXbyfzJS6ZGhbddGkRegdwImmfSVqCkEziYzrIGZ/TlrzBND4FysfkDg==", + "version": "1.4.341", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.341.tgz", + "integrity": "sha512-R4A8VfUBQY9WmAhuqY5tjHRf5fH2AAf6vqitBOE0y6u2PgHgqHSrhZmu78dIX3fVZtjqlwJNX1i2zwC3VpHtQQ==", "dev": true }, "node_modules/emojis-list": { @@ -4775,9 +4789,9 @@ } }, "@types/yargs": { - "version": "17.0.23", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.23.tgz", - "integrity": "sha512-yuogunc04OnzGQCrfHx+Kk883Q4X0aSwmYZhKjI21m+SVYzjIbrWl8dOOwSv5hf2Um2pdCOXWo9isteZTNXUZQ==", + "version": "17.0.24", + "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.24.tgz", + "integrity": "sha512-6i0aC7jV6QzQB8ne1joVZ0eSFIstHsCrobmOtghM11yGlH0j43FKL2UhWdELkyps0zuf7qVTUVCCR+tgSlyLLw==", "dev": true, "requires": { "@types/yargs-parser": "*" @@ -5065,6 +5079,12 @@ "integrity": "sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw==", "dev": true }, + "axe-core": { + "version": "4.6.3", + "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.6.3.tgz", + "integrity": "sha512-/BQzOX780JhsxDnPpH4ZiyrJAzcd8AfzFPkv+89veFSr1rcMjuq2JDCwypKaPeB6ljHp9KjXhPpjgCvQlWYuqg==", + "dev": true + }, "big.js": { "version": "5.2.2", "resolved": "https://registry.npmjs.org/big.js/-/big.js-5.2.2.tgz", @@ -5155,9 +5175,9 @@ } }, "caniuse-lite": { - "version": "1.0.30001470", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001470.tgz", - "integrity": "sha512-065uNwY6QtHCBOExzbV6m236DDhYCCtPmQUCoQtwkVqzud8v5QPidoMr6CoMkC2nfp6nksjttqWQRRh75LqUmA==", + "version": "1.0.30001472", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001472.tgz", + "integrity": "sha512-xWC/0+hHHQgj3/vrKYY0AAzeIUgr7L9wlELIcAvZdDUHlhL/kNxMdnQLOSOQfP8R51ZzPhmHdyMkI0MMpmxCfg==", "dev": true }, "chalk": { @@ -5839,9 +5859,9 @@ } }, "electron-to-chromium": { - "version": "1.4.340", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.340.tgz", - "integrity": "sha512-zx8hqumOqltKsv/MF50yvdAlPF9S/4PXbyfzJS6ZGhbddGkRegdwImmfSVqCkEziYzrIGZ/TlrzBND4FysfkDg==", + "version": "1.4.341", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.341.tgz", + "integrity": "sha512-R4A8VfUBQY9WmAhuqY5tjHRf5fH2AAf6vqitBOE0y6u2PgHgqHSrhZmu78dIX3fVZtjqlwJNX1i2zwC3VpHtQQ==", "dev": true }, "emojis-list": { diff --git a/package.json b/package.json index 4422ccb5d..f615c4a97 100644 --- a/package.json +++ b/package.json @@ -5,6 +5,7 @@ "build": "webpack" }, "devDependencies": { + "axe-core": "^4.6.3", "copy-webpack-plugin": "^11.0.0", "css-loader": "^3.4.2", "css-minimizer-webpack-plugin": "^4.2.2", diff --git a/pyproject.toml b/pyproject.toml index 02df26e15..bd32ec296 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,6 @@ name = "pydata-sphinx-theme" description = "Bootstrap-based Sphinx theme from the PyData community" dynamic = ["version"] readme = "README.md" - requires-python = ">=3.7" dependencies = [ "sphinx>=4.2", @@ -30,12 +29,10 @@ dependencies = [ "accessible-pygments", "typing-extensions" ] - license = { file = "LICENSE" } maintainers = [ { name = "Joris Van den Bossche", email = "jorisvandenbossche@gmail.com" }, ] - classifiers = [ "Development Status :: 5 - Production/Stable", "Programming Language :: Python :: 3", @@ -54,7 +51,7 @@ classifiers = [ doc = [ "numpydoc", "myst-nb", - "linkify-it-py", # for link shortening + "linkify-it-py", # for link shortening "rich", "sphinxext-rediraffe", "sphinx-sitemap", @@ -77,17 +74,9 @@ doc = [ "ipyleaflet", "colorama", ] -test = [ - "pytest", - "pytest-cov", - "pytest-regressions", -] -dev = [ - "pyyaml", - "pre-commit", - "nox", - "pydata-sphinx-theme[doc,test]", -] +test = ["pytest", "pytest-cov", "pytest-regressions"] +dev = ["pyyaml", "pre-commit", "nox", "pydata-sphinx-theme[doc,test]"] +a11y = ["pytest-playwright"] [project.entry-points] "sphinx.html_themes" = { pydata_sphinx_theme = "pydata_sphinx_theme" } @@ -100,8 +89,9 @@ ignore-init-module-imports = true fix = true select = ["E", "F", "W", "I", "D", "RUF"] ignore = [ - "E501", # line too long | Black take care of it - "D107", # Missing docstring in `__init__` | set the docstring in the class + "E501", # line too long | Black take care of it + "D107", # Missing docstring in `__init__` | set the docstring in the class + ] [tool.ruff.flake8-quotes] @@ -119,3 +109,6 @@ use_gitignore = true format_js = true format_css = true ignore = "H006,J018,T003,H025" + +[tool.pytest.ini_options] +markers = "a11y: mark a test as an accessibility test" diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 000000000..6608c4815 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,28 @@ +# PyData Sphinx tests + +This directory contains the Python tests for the theme. These tests are built with [pytest](https://docs.pytest.org/en/stable/) and are called through `nox`. + +- `test_build.py` checks that the static HTML output of the build process conforms + to various expectations. It builds static HTML pages based on configurations in + the `sites/` directory. The tests run various assertions on the static HTML + output, including snapshot comparisons with previously compiled outputs that are + stored in `test_build/`. In other words, it uses + [`pytest-regressions`](https://pytest-regressions.readthedocs.io/) to compare + the output created during the test run with a previously known and verified output + (stored under `test_build`) to make sure nothing has changed. + +- `test_a11y.py` checks PyData Sphinx Theme components for accessibility issues. + It's important to note that [only a fraction of accessibility issues can be + caught with automated + testing](https://accessibility.blog.gov.uk/2017/02/24/what-we-found-when-we-tested-tools-on-the-worlds-least-accessible-webpage/). + In contrast to the build test suite, the accessibility suite checks components as + they appear in the browser, meaning with any CSS and JavaScript applied. It does + this by building the PyData Sphinx Theme docs, launching a local server to the + docs, then checking the "Kitchen Sink" example pages with + [Playwright](https://playwright.dev), a program for developers that allows + loading and manipulating pages with various browsers, such as Chrome (chromium), + Firefox (gecko), Safari (WebKit). + +The ["Kitchen Sink" examples](https://pydata-sphinx-theme.readthedocs.io/en/stable/examples/kitchen-sink/index.html) +are taken from [sphinx-themes.org](https://sphinx-themes.org/) and showcase +components of the PyData Sphinx Theme, such as admonitions, lists, and headings. diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 000000000..cb0779c65 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +"""Tests for pydata-sphinx-theme.""" diff --git a/tests/test_a11y.py b/tests/test_a11y.py new file mode 100644 index 000000000..d007724a6 --- /dev/null +++ b/tests/test_a11y.py @@ -0,0 +1,112 @@ +"""Using Axe-core, scan the Kitchen Sink pages for accessibility violations.""" + +import time +from http.client import HTTPConnection +from pathlib import Path +from subprocess import PIPE, Popen +from urllib.parse import urljoin + +import pytest + +from .utils.pretty_axe_results import pretty_axe_results + +# Important note: automated accessibility scans can only find a fraction of +# potential accessibility issues. +# +# This test file scans pages from the Kitchen Sink examples with a JavaScript +# library called Axe-core, which checks the page for accessibility violations, +# such as places on the page with poor color contrast that would be hard for +# people with low vision to see. +# +# Just because a page passes the scan with no accessibility violations does +# *not* mean that it will be generally usable by a broad range of disabled +# people. It just means that page is free of common testable accessibility +# pitfalls. + +path_repo = Path(__file__).parent.parent +path_docs_build = path_repo / "docs" / "_build" / "html" + + +@pytest.fixture(scope="module") +def url_base(): + """Start local server on built docs and return the localhost URL as the base URL.""" + # Use a port that is not commonly used during development or else you will + # force the developer to stop running their dev server in order to run the + # tests. + port = "8213" + host = "localhost" + url = f"http://{host}:{port}" + + # Try starting the server + process = Popen( + ["python", "-m", "http.server", port, "--directory", path_docs_build], + stdout=PIPE, + ) + + # Try connecting to the server + retries = 5 + while retries > 0: + conn = HTTPConnection(host, port) + try: + conn.request("HEAD", "/") + response = conn.getresponse() + if response is not None: + yield url + break + except ConnectionRefusedError: + time.sleep(1) + retries -= 1 + + # If the code above never yields a URL, then we were never able to connect + # to the server and retries == 0. + if not retries: + raise RuntimeError("Failed to start http server in 5 seconds") + else: + # Otherwise the server started and this fixture is done now and we clean + # up by stopping the server. + process.terminate() + process.wait() + + +@pytest.mark.a11y +@pytest.mark.parametrize("theme", ["light", "dark"]) +@pytest.mark.parametrize( + "url_page,selector", + [ + ("admonitions.html", "#admonitions"), + ("api.html", "#api-documentation"), + ("blocks.html", "#blocks"), + ("generic.html", "#generic-items"), + ("images.html", "#images-figures"), + ("lists.html", "#lists"), + ("structure.html", "#structural-elements"), + ("structure.html", "#structural-elements-2"), + ("tables.html", "#tables"), + ("typography.html", "#typography"), + ], +) +def test_axe_core_kitchen_sink(theme: str, url_base: str, url_page: str, selector: str): + """Should have no Axe-core violations at the provided theme and page section.""" + # Using importorskip ensures that the test is skipped if not running within + # the a11y session + page = pytest.importorskip("playwright.sync_api.Page") + + # Load the page at the provided path + url_base_kitchen_sink = urljoin(url_base, "/examples/kitchen-sink/") + url_full = urljoin(url_base_kitchen_sink, url_page) + page.goto(url_full) + + # Run a line of JavaScript that sets the light/dark theme on the page + page.evaluate(f"document.documentElement.dataset.theme = '{theme}'") + + # Inject the Axe-core JavaScript library into the page + page.add_script_tag(path="node_modules/axe-core/axe.min.js") + + # Run the Axe-core library against a section of the page. (Don't run it + # against the whole page because in this test we're not trying to find + # accessibility violations in the nav, sidebar, footer, or other parts of + # the PyData Sphinx Theme documentation website.) + results = page.evaluate(f"axe.run('{selector}')") + + # Expect Axe-core to have found 0 accessibility violations + assert len(results["violations"]) == 0, pretty_axe_results(results) diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py new file mode 100644 index 000000000..a9abe4c5f --- /dev/null +++ b/tests/utils/__init__.py @@ -0,0 +1 @@ +"""Utility functions for testing pydata-sphinx-theme.""" diff --git a/tests/check_warnings.py b/tests/utils/check_warnings.py similarity index 96% rename from tests/check_warnings.py rename to tests/utils/check_warnings.py index 241f4acf1..9e790b835 100644 --- a/tests/check_warnings.py +++ b/tests/utils/check_warnings.py @@ -26,7 +26,7 @@ def check_warnings(file: Path) -> bool: print("\n=== Sphinx Warnings test ===\n") # find the file where all the known warnings are stored - warning_file = Path(__file__).parent / "warning_list.txt" + warning_file = Path(__file__).parent.parent / "warning_list.txt" test_warnings = file.read_text().strip().split("\n") ref_warnings = warning_file.read_text().strip().split("\n") diff --git a/tests/utils/pretty_axe_results.py b/tests/utils/pretty_axe_results.py new file mode 100644 index 000000000..698d42efd --- /dev/null +++ b/tests/utils/pretty_axe_results.py @@ -0,0 +1,46 @@ +"""Readable report of accessibility violations.""" + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This code taken from: +# https://github.com/mozilla-services/axe-selenium-python/blob/3cfbdd67c9b40ab03f37b3ba2521f77c2071827b/axe_selenium_python/axe.py +# Typical usage: +# assert(len(results["violations"])) == 0, pretty_axe_results(results) + + +def pretty_axe_results(results: dict) -> str: + """Create readable string that can be printed to console from the Axe-core results object. + + :param results: The object promised by `axe.run()`. + :type results: dict + :return report: Readable report of violations. + :rtype: string. + + """ + violations = results["violations"] + string = "" + string += f"Found {len(violations)} accessibility violations:" + for violation in violations: + string += ( + f"\n\n\nRule Violated:\n {violation['id']} - {violation['description']} \n\t" + f"URL: {violation['helpUrl']} \n\t" + f"Impact Level: {violation['impact']} \n\tTags:" + ) + for tag in violation["tags"]: + string += f" {tag}" + string += "\n\tElements Affected:" + i = 1 + for node in violation["nodes"]: + for target in node["target"]: + string += f"\n\t {i} Target: {target}" + i += 1 + for item in node["all"]: + string += f"\n\t\t {item['message']}" + for item in node["any"]: + string += f"\n\t\t {item['message']}" + for item in node["none"]: + string += f"\n\t\t {item['message']}" + string += "\n\n\n" + return string