diff --git a/docs/use/execute.md b/docs/use/execute.md index 3c23493c..c18cdc70 100644 --- a/docs/use/execute.md +++ b/docs/use/execute.md @@ -161,6 +161,12 @@ tags: [raises-exception] print(thisvariabledoesntexist) ``` +(execute/fail_on_error)= +### Error Reporting: Warning vs. Failure +When an error occurs in a context where `allow_errors=False`, the default behaviour is for this to be reported as a warning. This warning will simply be logged and not cause the build to fail unless `sphinx-build` is run with the [`-W` option](https://www.sphinx-doc.org/en/master/man/sphinx-build.html#cmdoption-sphinx-build-W). + +If you would like unexpected execution errors to cause a build failure rather than a warning regardless of the `-W` option, you can achieve this by setting `execution_fail_on_error=True` in your `conf.py`. + (execute/statistics)= ## Execution statistics diff --git a/docs/use/start.md b/docs/use/start.md index f4745fcb..01250dd8 100644 --- a/docs/use/start.md +++ b/docs/use/start.md @@ -77,6 +77,10 @@ Firstly for execution: - `False` - If `False`, when a code cell raises an error the execution is stopped, if `True` then all cells are always run. This can also be overridden by metadata in a notebook, [see here](execute/allow_errors) for details. +* - `execution_fail_on_error` + - `False` + - If `False`, disallowed errors will result in a sphinx warnings. + If `True`, then disallowed errors result in a failed build. [See here](execute/fail_on_error) for details. * - `execution_timeout` - 30 - The maximum time (in seconds) each notebook cell is allowed to run. diff --git a/myst_nb/__init__.py b/myst_nb/__init__.py index 50f34c2e..ccbd4ecd 100644 --- a/myst_nb/__init__.py +++ b/myst_nb/__init__.py @@ -125,6 +125,7 @@ def visit_element_html(self, node): app.add_config_value("jupyter_execute_notebooks", "auto", "env") app.add_config_value("execution_timeout", 30, "env") app.add_config_value("execution_allow_errors", False, "env") + app.add_config_value("execution_fail_on_error", False, "env") app.add_config_value("execution_in_temp", False, "env") # show traceback in stdout (in addition to writing to file) # this is useful in e.g. RTD where one cannot inspect a file diff --git a/myst_nb/execution.py b/myst_nb/execution.py index f5107e27..23ec5cb7 100644 --- a/myst_nb/execution.py +++ b/myst_nb/execution.py @@ -31,6 +31,10 @@ LOGGER = logging.getLogger(__name__) +class ExecutionError(Exception): + pass + + def update_execution_cache( app: Sphinx, builder: Builder, added: Set[str], changed: Set[str], removed: Set[str] ): @@ -80,6 +84,7 @@ def update_execution_cache( path_to_cache=app.env.nb_path_to_cache, timeout=app.config["execution_timeout"], allow_errors=app.config["execution_allow_errors"], + fail_on_error=app.config["execution_fail_on_error"], exec_in_temp=app.config["execution_in_temp"], ) @@ -144,14 +149,19 @@ def generate_notebook_outputs( report_path = None if result.err: - report_path, message = _report_exec_fail( - env, - Path(file_path).name, - result.exc_string, - show_traceback, - "Execution Failed with traceback saved in {}", - ) - LOGGER.error(message) + if env.config["execution_fail_on_error"]: + raise ExecutionError( + f"Execution failed for file: {file_path}\n{str(result.err)}" + ) + else: + report_path, message = _report_exec_fail( + env, + Path(file_path).name, + result.exc_string, + show_traceback, + "Execution Failed with traceback saved in {}", + ) + LOGGER.error(message) ntbk = result.nb @@ -197,7 +207,10 @@ def generate_notebook_outputs( ) message += suffix - LOGGER.error(message) + if env.config["execution_fail_on_error"]: + raise ExecutionError(message) + else: + LOGGER.error(message) else: LOGGER.verbose("Merged cached outputs into %s", str(r_file_path)) @@ -257,6 +270,7 @@ def _stage_and_execute( path_to_cache: str, timeout: Optional[int], allow_errors: bool, + fail_on_error: bool, exec_in_temp: bool, ): pk_list = [] @@ -280,6 +294,7 @@ def _stage_and_execute( timeout=timeout, exec_in_temp=exec_in_temp, allow_errors=allow_errors, + fail_on_error=fail_on_error, env=env, ) except OSError as err: @@ -289,9 +304,13 @@ def _stage_and_execute( # Normally we want to keep the stage records available, so that we can retrieve # execution tracebacks at the `generate_notebook_outputs` stage, # but we need to flush if it becomes 'corrupted' - LOGGER.error( - "Execution failed in an unexpected way, clearing staged notebooks: %s", err + message = ( + f"Execution failed in an unexpected way, clearing staged notebooks: {err}" ) + if fail_on_error: + raise ExecutionError(message) + else: + LOGGER.error(message) for record in cache_base.list_staged_records(): cache_base.discard_staged_notebook(record.pk) @@ -302,13 +321,17 @@ def execute_staged_nb( timeout: Optional[int], exec_in_temp: bool, allow_errors: bool, + fail_on_error: bool, env: BuildEnvironment, ): """Executing the staged notebook.""" try: executor = load_executor("basic", cache_base, logger=LOGGER) except ImportError as error: - LOGGER.error(str(error)) + if fail_on_error: + raise ExecutionError(str(error)) + else: + LOGGER.error(str(error)) return 1 def _converter(path): diff --git a/tests/test_execute.py b/tests/test_execute.py index 42216438..341a323e 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -1,6 +1,8 @@ import os import pytest +from myst_nb.execution import ExecutionError + def regress_nb_doc(file_regression, sphinx_run, check_nbs): file_regression.check( @@ -300,3 +302,27 @@ def test_custom_convert_cache(sphinx_run, file_regression, check_nbs): assert "custom-formats" in sphinx_run.env.nb_execution_data assert sphinx_run.env.nb_execution_data["custom-formats"]["method"] == "cache" assert sphinx_run.env.nb_execution_data["custom-formats"]["succeeded"] is True + + +@pytest.mark.sphinx_params( + "basic_failing.ipynb", + conf={"execution_allow_errors": False, "execution_fail_on_error": True}, +) +def test_execution_fail_on_error(sphinx_run, file_regression, check_nbs): + with pytest.raises(ExecutionError) as excinfo: + sphinx_run.build() + assert str(excinfo.value).startswith("Execution failed for file:") + # Ensure filename is reported: + assert "basic_failing.ipynb" in str(excinfo.value) + # Ensure failing code is reported: + assert "raise Exception('oopsie!')" in str(excinfo.value) + + +@pytest.mark.sphinx_params( + "basic_failing.ipynb", + conf={"execution_allow_errors": True, "execution_fail_on_error": True}, +) +def test_execution_fail_on_error_allow_errors(sphinx_run, file_regression, check_nbs): + sphinx_run.build() + assert not sphinx_run.warnings() + regress_nb_doc(file_regression, sphinx_run, check_nbs) diff --git a/tests/test_execute/test_execution_fail_on_error_allow_errors.ipynb b/tests/test_execute/test_execution_fail_on_error_allow_errors.ipynb new file mode 100644 index 00000000..2278726f --- /dev/null +++ b/tests/test_execute/test_execution_fail_on_error_allow_errors.ipynb @@ -0,0 +1,56 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# a title\n", + "\n", + "some text\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "ename": "Exception", + "evalue": "oopsie!", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mException\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'oopsie!'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;31mException\u001b[0m: oopsie!" + ] + } + ], + "source": [ + "raise Exception('oopsie!')" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.2" + }, + "test_name": "notebook1" + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/test_execute/test_execution_fail_on_error_allow_errors.xml b/tests/test_execute/test_execution_fail_on_error_allow_errors.xml new file mode 100644 index 00000000..9dd3813e --- /dev/null +++ b/tests/test_execute/test_execution_fail_on_error_allow_errors.xml @@ -0,0 +1,12 @@ + +
+ + a title + <paragraph> + some text + <CellNode cell_type="code" classes="cell"> + <CellInputNode classes="cell_input"> + <literal_block language="ipython3" xml:space="preserve"> + raise Exception('oopsie!') + <CellOutputNode classes="cell_output"> + <CellOutputBundleNode output_count="1">