From 8c4ef73a6515b00ddf2269e1038faeb61b28fc4c Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Mon, 17 Jul 2023 07:56:18 -0600 Subject: [PATCH 1/4] Add infrastructure for building tutorials This first commit is a rebase of Eric's initial PR as of db1ce6254 onto `main`, fixing up some changes caused by the CI infrastructure changing a bit since the PR was first opened. Co-authored-by: Jake Lishman --- .azure/docs-linux.yml | 2 +- .azure/tutorials-linux.yml | 40 ++++++++--------- .gitignore | 1 + docs/conf.py | 44 ++++++++++++++++--- docs/index.rst | 1 + docs/tutorials.rst | 42 ++++++++++++++++++ docs/tutorials/algorithms/placeholder.ipynb | 34 ++++++++++++++ docs/tutorials/circuits/placeholder.ipynb | 34 ++++++++++++++ .../circuits_advanced/placeholder.ipynb | 34 ++++++++++++++ docs/tutorials/operators/placeholder.ipynb | 34 ++++++++++++++ requirements-dev.txt | 9 ++-- requirements-tutorials.txt | 2 +- tox.ini | 16 +++++++ 13 files changed, 258 insertions(+), 35 deletions(-) create mode 100644 docs/tutorials.rst create mode 100644 docs/tutorials/algorithms/placeholder.ipynb create mode 100644 docs/tutorials/circuits/placeholder.ipynb create mode 100644 docs/tutorials/circuits_advanced/placeholder.ipynb create mode 100644 docs/tutorials/operators/placeholder.ipynb diff --git a/.azure/docs-linux.yml b/.azure/docs-linux.yml index 24c539b5e76..6e4e3049165 100644 --- a/.azure/docs-linux.yml +++ b/.azure/docs-linux.yml @@ -24,7 +24,7 @@ jobs: python -m pip install --upgrade pip setuptools wheel python -m pip install -U "tox<4.4.0" sudo apt-get update - sudo apt-get install -y graphviz + sudo apt-get install -y graphviz pandoc displayName: 'Install dependencies' - bash: | diff --git a/.azure/tutorials-linux.yml b/.azure/tutorials-linux.yml index 3e8b3678cab..80328bd3dbe 100644 --- a/.azure/tutorials-linux.yml +++ b/.azure/tutorials-linux.yml @@ -8,9 +8,7 @@ jobs: pool: {vmImage: 'ubuntu-latest'} variables: - QISKIT_SUPPRESS_PACKAGING_WARNINGS: Y PIP_CACHE_DIR: $(Pipeline.Workspace)/.pip - QISKIT_CELL_TIMEOUT: 300 steps: - task: UsePythonVersion@0 @@ -20,40 +18,38 @@ jobs: - bash: | set -e - git clone https://github.com/Qiskit/qiskit-tutorials --depth=1 python -m pip install --upgrade pip setuptools wheel - python -m pip install -U \ - -c constraints.txt \ - -r requirements.txt \ - -r requirements-dev.txt \ - -r requirements-optional.txt \ - -r requirements-tutorials.txt \ - -e . + python -m pip install -U "tox<4.4.0" sudo apt-get update sudo apt-get install -y graphviz pandoc - pip check + + git clone https://github.com/Qiskit/qiskit-tutorials --depth=1 + mv qiskit-tutorials/tutorials/algorithms/** docs/tutorials/algorithms + mv qiskit-tutorials/tutorials/circuits/** docs/tutorials/circuits + mv qiskit-tutorials/tutorials/circuits_advanced/** docs/tutorials/circuits_advanced + mv qiskit-tutorials/tutorials/operators/** docs/tutorials/operators displayName: 'Install dependencies' - env: - SETUPTOOLS_ENABLE_FEATURES: "legacy-editable" - bash: | - set -e - cd qiskit-tutorials - sphinx-build -b html . _build/html - env: - QISKIT_PARALLEL: False + tox -e tutorials + mkdir -p updated-tutorials/{algorithms,circuits,circuits_advanced,operators} + mv docs/tutorials/algorithms/*.nbconvert.ipynb updated-tutorials/algorithms/ + mv docs/tutorials/circuits/*.nbconvert.ipynb updated-tutorials/circuits/ + mv docs/tutorials/circuits_advanced/*.nbconvert.ipynb updated-tutorials/circuits_advanced/ + mv docs/tutorials/operators/*.nbconvert.ipynb updated-tutorials/operators/ + displayName: "Execute tutorials" - task: ArchiveFiles@2 inputs: - rootFolderOrFile: 'qiskit-tutorials/_build/html' + rootFolderOrFile: 'updated-tutorials' archiveType: tar - archiveFile: '$(Build.ArtifactStagingDirectory)/html_tutorials.tar.gz' + archiveFile: '$(Build.ArtifactStagingDirectory)/updated-tutorials.tar.gz' verbose: true - task: PublishBuildArtifacts@1 - displayName: 'Publish docs' + displayName: 'Publish updated tutorials' inputs: pathtoPublish: '$(Build.ArtifactStagingDirectory)' - artifactName: 'html_tutorials' + artifactName: 'updated-tutorials' Parallel: true ParallelCount: 8 diff --git a/.gitignore b/.gitignore index d131395fe50..4cbf6c1cbe0 100644 --- a/.gitignore +++ b/.gitignore @@ -143,6 +143,7 @@ qiskit/transpiler/passes/**/cython/**/*.cpp qiskit/quantum_info/states/cython/*.cpp docs/stubs/* +*.nbconvert.ipynb # Notebook testing images test/visual/mpl/circuit/circuit_results/*.png diff --git a/docs/conf.py b/docs/conf.py index 1c5292a64c8..94e60f85ddd 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -14,12 +14,12 @@ """Sphinx documentation builder.""" -# -- General configuration --------------------------------------------------- import datetime import doctest +import os project = "Qiskit" -copyright = f"2017-{datetime.date.today().year}, Qiskit Development Team" # pylint: disable=redefined-builtin +project_copyright = f"2017-{datetime.date.today().year}, Qiskit Development Team" author = "Qiskit Development Team" # The short X.Y version @@ -39,7 +39,8 @@ "reno.sphinxext", "sphinx_design", "matplotlib.sphinxext.plot_directive", - "sphinx.ext.doctest", + "qiskit_sphinx_theme", + "nbsphinx", ] templates_path = ["_templates"] @@ -77,7 +78,9 @@ "matplotlib": ("https://matplotlib.org/stable/", None), } -# -- Options for HTML output ------------------------------------------------- +# ---------------------------------------------------------------------------------- +# HTML theme +# ---------------------------------------------------------------------------------- html_theme = "qiskit_sphinx_theme" html_last_updated_fmt = "%Y/%m/%d" @@ -88,8 +91,9 @@ "style_external_links": True, } - -# -- Options for Autosummary and Autodoc ------------------------------------- +# ---------------------------------------------------------------------------------- +# Autodoc +# ---------------------------------------------------------------------------------- # Note that setting autodoc defaults here may not have as much of an effect as you may expect; any # documentation created by autosummary uses a template file (in autosummary in the templates path), @@ -131,7 +135,9 @@ napoleon_numpy_docstring = False -# -- Options for Doctest -------------------------------------------------------- +# ---------------------------------------------------------------------------------- +# Doctest +# ---------------------------------------------------------------------------------- doctest_default_flags = ( doctest.ELLIPSIS @@ -145,3 +151,27 @@ # >> code # output doctest_test_doctest_blocks = "" + +# ---------------------------------------------------------------------------------- +# Nbsphinx +# ---------------------------------------------------------------------------------- + +nbsphinx_timeout = 300 +nbsphinx_execute = os.getenv("QISKIT_DOCS_BUILD_TUTORIALS", "never") +nbsphinx_widgets_path = "" +nbsphinx_thumbnails = {"**": "_static/images/logo.png"} + +nbsphinx_prolog = """ +{% set docname = env.doc2path(env.docname, base=None) %} + +.. only:: html + + .. role:: raw-html(raw) + :format: html + + .. note:: + This page was generated from `{{ docname }}`__. + + __ https://github.com/Qiskit/qiskit-terra/blob/main/{{ docname }} + +""" diff --git a/docs/index.rst b/docs/index.rst index 903fc64030c..51de241b235 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -7,6 +7,7 @@ Qiskit Terra documentation :hidden: How-to Guides + tutorials API References Explanation Migration Guides diff --git a/docs/tutorials.rst b/docs/tutorials.rst new file mode 100644 index 00000000000..7c80c3a48bf --- /dev/null +++ b/docs/tutorials.rst @@ -0,0 +1,42 @@ +.. _tutorials: + +========= +Tutorials +========= + +Quantum circuits +================ + +.. nbgallery:: + :glob: + + tutorials/circuits/* + +Advanced circuits +================= + +.. nbgallery:: + :glob: + + tutorials/circuits_advanced/* + +Algorithms +========== + +.. nbgallery:: + :glob: + + tutorials/algorithms/* + +Operators +========= + +.. nbgallery:: + :glob: + + tutorials/operators/* + +.. Hiding - Indices and tables + :ref:`genindex` + :ref:`modindex` + :ref:`search` diff --git a/docs/tutorials/algorithms/placeholder.ipynb b/docs/tutorials/algorithms/placeholder.ipynb new file mode 100644 index 00000000000..6f1c1caae82 --- /dev/null +++ b/docs/tutorials/algorithms/placeholder.ipynb @@ -0,0 +1,34 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Placeholder", + "\n", + "This is only here to test the infrastructure for tutorials. It will be removed with our actual tutorials from qiskit-tutorials once finishing the metapackage migration." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.16" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/docs/tutorials/circuits/placeholder.ipynb b/docs/tutorials/circuits/placeholder.ipynb new file mode 100644 index 00000000000..6f1c1caae82 --- /dev/null +++ b/docs/tutorials/circuits/placeholder.ipynb @@ -0,0 +1,34 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Placeholder", + "\n", + "This is only here to test the infrastructure for tutorials. It will be removed with our actual tutorials from qiskit-tutorials once finishing the metapackage migration." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.16" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/docs/tutorials/circuits_advanced/placeholder.ipynb b/docs/tutorials/circuits_advanced/placeholder.ipynb new file mode 100644 index 00000000000..6f1c1caae82 --- /dev/null +++ b/docs/tutorials/circuits_advanced/placeholder.ipynb @@ -0,0 +1,34 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Placeholder", + "\n", + "This is only here to test the infrastructure for tutorials. It will be removed with our actual tutorials from qiskit-tutorials once finishing the metapackage migration." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.16" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/docs/tutorials/operators/placeholder.ipynb b/docs/tutorials/operators/placeholder.ipynb new file mode 100644 index 00000000000..6f1c1caae82 --- /dev/null +++ b/docs/tutorials/operators/placeholder.ipynb @@ -0,0 +1,34 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Placeholder", + "\n", + "This is only here to test the infrastructure for tutorials. It will be removed with our actual tutorials from qiskit-tutorials once finishing the metapackage migration." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.16" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/requirements-dev.txt b/requirements-dev.txt index b373c7b08c3..177180769d9 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -31,9 +31,10 @@ ddt>=1.2.0,!=1.4.0,!=1.4.3 # components of Terra use some of its optional dependencies in order to document # themselves. These are the requirements that are _only_ required for the docs # build, and are not used by Terra itself. - +Sphinx>=6.0 +qiskit-sphinx-theme~=1.13.0 +sphinx-design>=0.2.0 +nbsphinx~=0.9.2 +nbconvert~=7.7.1 # TODO: switch to stable release when 4.1 is released reno @ git+https://github.com/openstack/reno.git@81587f616f17904336cdc431e25c42b46cd75b8f -Sphinx>=5.0 -qiskit-sphinx-theme~=1.11.0 -sphinx-design>=0.2.0 diff --git a/requirements-tutorials.txt b/requirements-tutorials.txt index 5aa9d0c412c..c87701dc97a 100644 --- a/requirements-tutorials.txt +++ b/requirements-tutorials.txt @@ -2,7 +2,7 @@ # This may also include some requirements that are only in `requirements-dev.txt`, since those # aren't runtime dependencies or optionals of Terra. -networkx>=2.2 +networkx>=2.3 jupyter Sphinx nbsphinx diff --git a/tox.ini b/tox.ini index 36fb269f613..7cfd5733876 100644 --- a/tox.ini +++ b/tox.ini @@ -73,6 +73,8 @@ setenv = {[testenv]setenv} QISKIT_SUPPRESS_PACKAGING_WARNINGS=Y RUST_DEBUG=1 # Faster to compile. +passenv = + QISKIT_DOCS_BUILD_TUTORIALS deps = setuptools_rust # This is work around for the bug of tox 3 (see #8606 for more details.) -r{toxinidir}/requirements-dev.txt @@ -92,3 +94,17 @@ allowlist_externals = rm commands = rm -rf {toxinidir}/docs/stubs/ {toxinidir}/docs/_build + +[testenv:tutorials] +usedevelop = False +basepython = python3 +setenv = + {[testenv]setenv} + RUST_DEBUG=1 # Faster to compile. + PYDEVD_DISABLE_FILE_VALIDATION=1 +deps = + {[testenv:docs]deps} + -r{toxinidir}/requirements-tutorials.txt + networkx>=2.3 +commands = + jupyter nbconvert --to notebook --execute --ExecutePreprocessor.timeout=300 docs/tutorials/**/*.ipynb From 5ababc0e3ee68d3cfc599816e3ef8bae0d8a5e45 Mon Sep 17 00:00:00 2001 From: Jake Lishman Date: Mon, 7 Aug 2023 16:15:45 +0100 Subject: [PATCH 2/4] Harden tutorials Azure job This moves much of the fetch- and process-related code into separate scripts that assert far more about the directory structure, and fail if they do not match the assumptions. We don't want to accidentally get out-of-sync while we're changing things and end up with a tutorials job that isn't really doing its job without us noticing. The tutorials-fetching script can now also be re-used in a separate GitHub Actions workflow that will handle the full tutorials-included documentation build and deployment in the future. The notebook-convertion step is moved into Python space, using `nbconvert` as a library in order to parallelise the build process for the tutorials, and to allow CI and developers calling `tox` directly to specify the output directories for the built tutorials. --- .azure/tutorials-linux.yml | 19 +++----- docs/tutorials.rst | 5 --- tools/convert_tutorials.py | 87 ++++++++++++++++++++++++++++++++++++ tools/prepare_tutorials.bash | 61 +++++++++++++++++++++++++ tox.ini | 8 +--- 5 files changed, 155 insertions(+), 25 deletions(-) create mode 100644 tools/convert_tutorials.py create mode 100755 tools/prepare_tutorials.bash diff --git a/.azure/tutorials-linux.yml b/.azure/tutorials-linux.yml index 80328bd3dbe..83a0b928ac8 100644 --- a/.azure/tutorials-linux.yml +++ b/.azure/tutorials-linux.yml @@ -22,21 +22,12 @@ jobs: python -m pip install -U "tox<4.4.0" sudo apt-get update sudo apt-get install -y graphviz pandoc - - git clone https://github.com/Qiskit/qiskit-tutorials --depth=1 - mv qiskit-tutorials/tutorials/algorithms/** docs/tutorials/algorithms - mv qiskit-tutorials/tutorials/circuits/** docs/tutorials/circuits - mv qiskit-tutorials/tutorials/circuits_advanced/** docs/tutorials/circuits_advanced - mv qiskit-tutorials/tutorials/operators/** docs/tutorials/operators displayName: 'Install dependencies' - - bash: | - tox -e tutorials - mkdir -p updated-tutorials/{algorithms,circuits,circuits_advanced,operators} - mv docs/tutorials/algorithms/*.nbconvert.ipynb updated-tutorials/algorithms/ - mv docs/tutorials/circuits/*.nbconvert.ipynb updated-tutorials/circuits/ - mv docs/tutorials/circuits_advanced/*.nbconvert.ipynb updated-tutorials/circuits_advanced/ - mv docs/tutorials/operators/*.nbconvert.ipynb updated-tutorials/operators/ + - bash: tools/prepare_tutorials.bash algorithms circuits circuits_advanced operators + displayName: 'Download current tutorials' + + - bash: tox -e tutorials -- --out updated-tutorials displayName: "Execute tutorials" - task: ArchiveFiles@2 @@ -45,6 +36,7 @@ jobs: archiveType: tar archiveFile: '$(Build.ArtifactStagingDirectory)/updated-tutorials.tar.gz' verbose: true + condition: succeededOrFailed() - task: PublishBuildArtifacts@1 displayName: 'Publish updated tutorials' @@ -53,3 +45,4 @@ jobs: artifactName: 'updated-tutorials' Parallel: true ParallelCount: 8 + condition: succeededOrFailed() diff --git a/docs/tutorials.rst b/docs/tutorials.rst index 7c80c3a48bf..5b230dd0e3a 100644 --- a/docs/tutorials.rst +++ b/docs/tutorials.rst @@ -35,8 +35,3 @@ Operators :glob: tutorials/operators/* - -.. Hiding - Indices and tables - :ref:`genindex` - :ref:`modindex` - :ref:`search` diff --git a/tools/convert_tutorials.py b/tools/convert_tutorials.py new file mode 100644 index 00000000000..7e3b5db5115 --- /dev/null +++ b/tools/convert_tutorials.py @@ -0,0 +1,87 @@ +# This code is part of Qiskit. +# +# (C) Copyright IBM 2023 +# +# This code is licensed under the Apache License, Version 2.0. You may +# obtain a copy of this license in the LICENSE.txt file in the root directory +# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0. +# +# Any modifications or derivative works of this code must retain this +# copyright notice, and modified files need to carry a notice indicating +# that they have been altered from the originals. + +# pylint: disable=missing-function-docstring,broad-exception-caught + +""" +Utility script to parallelise the conversion of several Jupyter notebooks. + +If nbconvert starts offering built-in parallelisation this script can likely be dropped. +""" + +import argparse +import multiprocessing +import os +import pathlib +import sys +import typing + +import nbformat +from nbconvert.preprocessors import ExecutePreprocessor + + +def worker( + notebook_path: pathlib.Path, in_root: pathlib.Path, out_root: typing.Optional[pathlib.Path] +) -> typing.Optional[Exception]: + """Single parallel worker that spawns a Jupyter executor node, executes the given notebook + within it, and writes out the output.""" + try: + print(f"({os.getpid()}) Processing '{str(notebook_path)}'", flush=True) + processor = ExecutePreprocessor(timeout=300, kernel_name="python3") + with open(notebook_path, "r") as fptr: + notebook = nbformat.read(fptr, as_version=4) + # Run the notebook with the working directory set to the folder it resides in. + processor.preprocess(notebook, {"metadata": {"path": f"{notebook_path.parent}/"}}) + + # Ensure the output directory exists, and write to it. + out_root = in_root if out_root is None else out_root + out_path = out_root / notebook_path.relative_to(in_root).with_suffix(".nbconvert.ipynb") + out_path.parent.mkdir(parents=True, exist_ok=True) + with open(out_path, "w", encoding="utf-8") as fptr: + nbformat.write(notebook, fptr) + except Exception as exc: + return exc + return None + + +def main() -> int: + parser = argparse.ArgumentParser(description="Execute tutorial Jupyter notebooks.") + parser.add_argument( + "notebook_dirs", type=pathlib.Path, nargs="*", help="Folders containing Jupyter notebooks." + ) + parser.add_argument( + "--out", + type=pathlib.Path, + help="Output directory for files. Defaults to same location as input file.", + ) + args = parser.parse_args() + notebooks = sorted( + { + (notebook_path, in_root, args.out) + for in_root in args.notebook_dirs + for notebook_path in in_root.glob("**/*.ipynb") + } + ) + cpus = os.cpu_count() + print(f"Using {cpus} processes.") + with multiprocessing.Pool(cpus) as pool: + failures = pool.starmap(worker, notebooks) + num_failures = 0 + for path, failure in zip(notebooks, failures): + if failure is not None: + print(f"'{path}' failed: {failure}", file=sys.stderr) + num_failures += 1 + return num_failures + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/prepare_tutorials.bash b/tools/prepare_tutorials.bash new file mode 100755 index 00000000000..bfa92942392 --- /dev/null +++ b/tools/prepare_tutorials.bash @@ -0,0 +1,61 @@ +#!/bin/bash +# +# Clone the tutorials in from Qiskit/qiskit-tutorials, and put them in the right +# place in the documentation structure ready for a complete documentation build. +# In the initial transition from the metapackage structure, we are leaving +# placeholder files in the documentation directories so everything will build +# happily without the full structure having been cloned, but this may change in +# the future. +# +# If the placeholder files are removed in the future, this script will likely +# have become obsolete and the CI pipelines (or this script) should be updated +# to reflect that. +# +# Usage: +# prepare_tutorials.sh components ... +# +# components +# Subdirectories of `tutorials` in the tutorials repository that should be +# moved into the correct locations in the source tree. + +set -e + +if [[ $# -eq 0 ]]; then + echo "Usage: prepare_tutorials.sh components ..." >&2 + exit 1 +fi + +# Pull in the tutorials repository. +tmpdir="$(mktemp -d)" +git clone --depth=1 https://github.com/Qiskit/qiskit-tutorials "$tmpdir" +indir="${tmpdir}/tutorials" + +outdir="$(dirname "$(dirname "${BASH_SOURCE[0]}")")/docs/tutorials" +if [[ ! -d "$outdir" ]]; then + echo "Tutorials documentation directory '${outdir}' does not exist." >&2 + exit 2 +fi + +for component in "$@"; do + echo "Getting tutorials from '${component}'" + + if [[ ! -d "${indir}/${component}" ]]; then + echo "Component '${component}' not in tutorials repository." >&2 + exit 3 + fi + if [[ -d "${outdir}/${component}" && -f "${outdir}/${component}/placeholder.ipynb" ]]; then + rm "${outdir}/${component}/placeholder.ipynb" + if [[ -z "$(ls -A "${outdir}/${component}")" ]]; then + rm -r "${outdir}/${component}" + else + echo "Directory '${outdir}/${component}' contains files other than the placeholder. This script needs updating." >&2 + exit 4 + fi + else + echo "Directory '${outdir}/${component}' does not exist, or has no placeholder. This script needs updating." >&2 + exit 5 + fi + mv "${indir}/${component}" "${outdir}/${component}" +done + +rm -rf "${tmpdir}" diff --git a/tox.ini b/tox.ini index 7cfd5733876..022a8e01e81 100644 --- a/tox.ini +++ b/tox.ini @@ -96,15 +96,9 @@ commands = rm -rf {toxinidir}/docs/stubs/ {toxinidir}/docs/_build [testenv:tutorials] -usedevelop = False basepython = python3 -setenv = - {[testenv]setenv} - RUST_DEBUG=1 # Faster to compile. - PYDEVD_DISABLE_FILE_VALIDATION=1 deps = {[testenv:docs]deps} -r{toxinidir}/requirements-tutorials.txt - networkx>=2.3 commands = - jupyter nbconvert --to notebook --execute --ExecutePreprocessor.timeout=300 docs/tutorials/**/*.ipynb + python tools/convert_tutorials.py {toxinidir}/docs/tutorials {posargs} From 3e4ad57d3735657a8b6eac489a1eccc6509aaa52 Mon Sep 17 00:00:00 2001 From: Jake Lishman Date: Mon, 7 Aug 2023 22:55:23 +0100 Subject: [PATCH 3/4] Retarget tutorial-conversion utility as executor This reorganises the tutorial "conversion" utility to make it clearer that what it's actually doing is just executing the tutorials. The script itself is changed to default to editing the files inplace, while the `tox` job is updated to write the files into a special directory, making it easier to clean up a dirty build directory and making it so subsequent local executions will not pick up the converted files. --- .azure/tutorials-linux.yml | 8 ++++---- .gitignore | 2 +- tools/{convert_tutorials.py => execute_tutorials.py} | 7 ++++--- tox.ini | 2 +- 4 files changed, 10 insertions(+), 9 deletions(-) rename tools/{convert_tutorials.py => execute_tutorials.py} (93%) diff --git a/.azure/tutorials-linux.yml b/.azure/tutorials-linux.yml index 83a0b928ac8..c8fd8d75cca 100644 --- a/.azure/tutorials-linux.yml +++ b/.azure/tutorials-linux.yml @@ -27,14 +27,14 @@ jobs: - bash: tools/prepare_tutorials.bash algorithms circuits circuits_advanced operators displayName: 'Download current tutorials' - - bash: tox -e tutorials -- --out updated-tutorials + - bash: tox -e tutorials displayName: "Execute tutorials" - task: ArchiveFiles@2 inputs: - rootFolderOrFile: 'updated-tutorials' + rootFolderOrFile: 'executed_tutorials' archiveType: tar - archiveFile: '$(Build.ArtifactStagingDirectory)/updated-tutorials.tar.gz' + archiveFile: '$(Build.ArtifactStagingDirectory)/executed_tutorials.tar.gz' verbose: true condition: succeededOrFailed() @@ -42,7 +42,7 @@ jobs: displayName: 'Publish updated tutorials' inputs: pathtoPublish: '$(Build.ArtifactStagingDirectory)' - artifactName: 'updated-tutorials' + artifactName: 'executed_tutorials' Parallel: true ParallelCount: 8 condition: succeededOrFailed() diff --git a/.gitignore b/.gitignore index 4cbf6c1cbe0..4a7757bb3d0 100644 --- a/.gitignore +++ b/.gitignore @@ -143,7 +143,7 @@ qiskit/transpiler/passes/**/cython/**/*.cpp qiskit/quantum_info/states/cython/*.cpp docs/stubs/* -*.nbconvert.ipynb +executed_tutorials/ # Notebook testing images test/visual/mpl/circuit/circuit_results/*.png diff --git a/tools/convert_tutorials.py b/tools/execute_tutorials.py similarity index 93% rename from tools/convert_tutorials.py rename to tools/execute_tutorials.py index 7e3b5db5115..383b8f0e233 100644 --- a/tools/convert_tutorials.py +++ b/tools/execute_tutorials.py @@ -42,9 +42,10 @@ def worker( # Run the notebook with the working directory set to the folder it resides in. processor.preprocess(notebook, {"metadata": {"path": f"{notebook_path.parent}/"}}) - # Ensure the output directory exists, and write to it. + # Ensure the output directory exists, and write to it. This overwrites the notebook with + # its executed form unless the '--out' flag was set. out_root = in_root if out_root is None else out_root - out_path = out_root / notebook_path.relative_to(in_root).with_suffix(".nbconvert.ipynb") + out_path = out_root / notebook_path.relative_to(in_root) out_path.parent.mkdir(parents=True, exist_ok=True) with open(out_path, "w", encoding="utf-8") as fptr: nbformat.write(notebook, fptr) @@ -61,7 +62,7 @@ def main() -> int: parser.add_argument( "--out", type=pathlib.Path, - help="Output directory for files. Defaults to same location as input file.", + help="Output directory for files. Defaults to same location as input file, overwriting it.", ) args = parser.parse_args() notebooks = sorted( diff --git a/tox.ini b/tox.ini index 022a8e01e81..9ccb4b87bff 100644 --- a/tox.ini +++ b/tox.ini @@ -101,4 +101,4 @@ deps = {[testenv:docs]deps} -r{toxinidir}/requirements-tutorials.txt commands = - python tools/convert_tutorials.py {toxinidir}/docs/tutorials {posargs} + python tools/execute_tutorials.py {toxinidir}/docs/tutorials --out={toxinidir}/executed_tutorials From 6e3f4ff957b44542af97c37b02bcabb6f536bb6d Mon Sep 17 00:00:00 2001 From: Jake Lishman Date: Wed, 9 Aug 2023 16:00:42 +0100 Subject: [PATCH 4/4] Allow configuration of tutorials execution There was a worry that not being able to configure these would make it more unpleasant to use `tox` for the jobs locally. --- .azure/tutorials-linux.yml | 2 ++ docs/conf.py | 2 +- tools/execute_tutorials.py | 24 ++++++++++++++++++------ tox.ini | 6 +++--- 4 files changed, 24 insertions(+), 10 deletions(-) diff --git a/.azure/tutorials-linux.yml b/.azure/tutorials-linux.yml index c8fd8d75cca..197ca186615 100644 --- a/.azure/tutorials-linux.yml +++ b/.azure/tutorials-linux.yml @@ -29,6 +29,8 @@ jobs: - bash: tox -e tutorials displayName: "Execute tutorials" + env: + QISKIT_CELL_TIMEOUT: 300 - task: ArchiveFiles@2 inputs: diff --git a/docs/conf.py b/docs/conf.py index 94e60f85ddd..4c3409ccf4c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -156,7 +156,7 @@ # Nbsphinx # ---------------------------------------------------------------------------------- -nbsphinx_timeout = 300 +nbsphinx_timeout = int(os.getenv("QISKIT_CELL_TIMEOUT", "300")) nbsphinx_execute = os.getenv("QISKIT_DOCS_BUILD_TUTORIALS", "never") nbsphinx_widgets_path = "" nbsphinx_thumbnails = {"**": "_static/images/logo.png"} diff --git a/tools/execute_tutorials.py b/tools/execute_tutorials.py index 383b8f0e233..2ce0c0c07fe 100644 --- a/tools/execute_tutorials.py +++ b/tools/execute_tutorials.py @@ -19,6 +19,7 @@ """ import argparse +import functools import multiprocessing import os import pathlib @@ -30,13 +31,16 @@ def worker( - notebook_path: pathlib.Path, in_root: pathlib.Path, out_root: typing.Optional[pathlib.Path] + notebook_path: pathlib.Path, + in_root: pathlib.Path, + out_root: typing.Optional[pathlib.Path], + timeout: int = -1, ) -> typing.Optional[Exception]: """Single parallel worker that spawns a Jupyter executor node, executes the given notebook within it, and writes out the output.""" try: print(f"({os.getpid()}) Processing '{str(notebook_path)}'", flush=True) - processor = ExecutePreprocessor(timeout=300, kernel_name="python3") + processor = ExecutePreprocessor(timeout=timeout, kernel_name="python3") with open(notebook_path, "r") as fptr: notebook = nbformat.read(fptr, as_version=4) # Run the notebook with the working directory set to the folder it resides in. @@ -60,10 +64,18 @@ def main() -> int: "notebook_dirs", type=pathlib.Path, nargs="*", help="Folders containing Jupyter notebooks." ) parser.add_argument( + "-o", "--out", type=pathlib.Path, help="Output directory for files. Defaults to same location as input file, overwriting it.", ) + parser.add_argument( + "-j", + "--num-processes", + type=int, + default=os.cpu_count(), + help="Number of processes to use.", + ) args = parser.parse_args() notebooks = sorted( { @@ -72,10 +84,10 @@ def main() -> int: for notebook_path in in_root.glob("**/*.ipynb") } ) - cpus = os.cpu_count() - print(f"Using {cpus} processes.") - with multiprocessing.Pool(cpus) as pool: - failures = pool.starmap(worker, notebooks) + timeout = int(os.getenv("QISKIT_CELL_TIMEOUT", "300")) + print(f"Using {args.num_processes} process{'' if args.num_processes == 1 else 'es'}.") + with multiprocessing.Pool(args.num_processes) as pool: + failures = pool.starmap(functools.partial(worker, timeout=timeout), notebooks) num_failures = 0 for path, failure in zip(notebooks, failures): if failure is not None: diff --git a/tox.ini b/tox.ini index 9ccb4b87bff..d62dcc9fa6c 100644 --- a/tox.ini +++ b/tox.ini @@ -73,8 +73,7 @@ setenv = {[testenv]setenv} QISKIT_SUPPRESS_PACKAGING_WARNINGS=Y RUST_DEBUG=1 # Faster to compile. -passenv = - QISKIT_DOCS_BUILD_TUTORIALS +passenv = {[testenv]passenv}, QISKIT_DOCS_BUILD_TUTORIALS deps = setuptools_rust # This is work around for the bug of tox 3 (see #8606 for more details.) -r{toxinidir}/requirements-dev.txt @@ -100,5 +99,6 @@ basepython = python3 deps = {[testenv:docs]deps} -r{toxinidir}/requirements-tutorials.txt +passenv = {[testenv]passenv}, QISKIT_CELL_TIMEOUT commands = - python tools/execute_tutorials.py {toxinidir}/docs/tutorials --out={toxinidir}/executed_tutorials + python tools/execute_tutorials.py {toxinidir}/docs/tutorials --out={toxinidir}/executed_tutorials {posargs}