✨ NEW: Add execution_fail_on_error configuration

docs/use/execute.md

@@ -161,6 +161,12 @@ tags: [raises-exception]
 print(thisvariabledoesntexist)
 ```
 
+(execute/fail_on_error)=
+### Error Reporting: Warning vs. Failure
+When an error occurs in a context where `allow_errors=False`, the default behaviour is for this to be reported as a warning. This warning will simply be logged and not cause the build to fail unless `sphinx-build` is run with the [`-W` option](https://www.sphinx-doc.org/en/master/man/sphinx-build.html#cmdoption-sphinx-build-W).
+
+If you would like unexpected execution errors to cause a build failure rather than a warning regardless of the `-W` option, you can achieve this by setting `execution_fail_on_error=True` in your `conf.py`.
+
 (execute/statistics)=
 ## Execution statistics
 

docs/use/start.md

@@ -77,6 +77,10 @@ Firstly for execution:
   - `False`
   - If `False`, when a code cell raises an error the execution is stopped, if `True` then all cells are always run.
     This can also be overridden by metadata in a notebook, [see here](execute/allow_errors) for details.
+* - `execution_fail_on_error`
+  - `False`
+  - If `False`, disallowed errors will result in a sphinx warnings.
+    If `True`, then disallowed errors result in a failed build. [See here](execute/fail_on_error) for details.
 * - `execution_timeout`
   - 30
   - The maximum time (in seconds) each notebook cell is allowed to run.

myst_nb/__init__.py

@@ -125,6 +125,7 @@ def visit_element_html(self, node):
     app.add_config_value("jupyter_execute_notebooks", "auto", "env")
     app.add_config_value("execution_timeout", 30, "env")
     app.add_config_value("execution_allow_errors", False, "env")
+    app.add_config_value("execution_fail_on_error", False, "env")
     app.add_config_value("execution_in_temp", False, "env")
     # show traceback in stdout (in addition to writing to file)
     # this is useful in e.g. RTD where one cannot inspect a file

myst_nb/execution.py

@@ -31,6 +31,10 @@
 LOGGER = logging.getLogger(__name__)
 
 
+class ExecutionError(Exception):
+    pass
+
+
 def update_execution_cache(
     app: Sphinx, builder: Builder, added: Set[str], changed: Set[str], removed: Set[str]
 ):
@@ -80,6 +84,7 @@ def update_execution_cache(
             path_to_cache=app.env.nb_path_to_cache,
             timeout=app.config["execution_timeout"],
             allow_errors=app.config["execution_allow_errors"],
+            fail_on_error=app.config["execution_fail_on_error"],
             exec_in_temp=app.config["execution_in_temp"],
         )
 
@@ -144,14 +149,19 @@ def generate_notebook_outputs(
 
             report_path = None
             if result.err:
-                report_path, message = _report_exec_fail(
-                    env,
-                    Path(file_path).name,
-                    result.exc_string,
-                    show_traceback,
-                    "Execution Failed with traceback saved in {}",
-                )
-                LOGGER.error(message)
+                if env.config["execution_fail_on_error"]:
+                    raise ExecutionError(
+                        f"Execution failed for file: {file_path}\n{str(result.err)}"
+                    )
+                else:
+                    report_path, message = _report_exec_fail(
+                        env,
+                        Path(file_path).name,
+                        result.exc_string,
+                        show_traceback,
+                        "Execution Failed with traceback saved in {}",
+                    )
+                    LOGGER.error(message)
 
             ntbk = result.nb
 
@@ -197,7 +207,10 @@ def generate_notebook_outputs(
             )
             message += suffix
 
-        LOGGER.error(message)
+        if env.config["execution_fail_on_error"]:
+            raise ExecutionError(message)
+        else:
+            LOGGER.error(message)
 
     else:
         LOGGER.verbose("Merged cached outputs into %s", str(r_file_path))
@@ -257,6 +270,7 @@ def _stage_and_execute(
     path_to_cache: str,
     timeout: Optional[int],
     allow_errors: bool,
+    fail_on_error: bool,
     exec_in_temp: bool,
 ):
     pk_list = []
@@ -280,6 +294,7 @@ def _stage_and_execute(
                 timeout=timeout,
                 exec_in_temp=exec_in_temp,
                 allow_errors=allow_errors,
+                fail_on_error=fail_on_error,
                 env=env,
             )
     except OSError as err:
@@ -289,9 +304,13 @@ def _stage_and_execute(
         # Normally we want to keep the stage records available, so that we can retrieve
         # execution tracebacks at the `generate_notebook_outputs` stage,
         # but we need to flush if it becomes 'corrupted'
-        LOGGER.error(
-            "Execution failed in an unexpected way, clearing staged notebooks: %s", err
+        message = (
+            f"Execution failed in an unexpected way, clearing staged notebooks: {err}"
         )
+        if fail_on_error:
+            raise ExecutionError(message)
+        else:
+            LOGGER.error(message)
         for record in cache_base.list_staged_records():
             cache_base.discard_staged_notebook(record.pk)
 
@@ -302,13 +321,17 @@ def execute_staged_nb(
     timeout: Optional[int],
     exec_in_temp: bool,
     allow_errors: bool,
+    fail_on_error: bool,
     env: BuildEnvironment,
 ):
     """Executing the staged notebook."""
     try:
         executor = load_executor("basic", cache_base, logger=LOGGER)
     except ImportError as error:
-        LOGGER.error(str(error))
+        if fail_on_error:
+            raise ExecutionError(str(error))
+        else:
+            LOGGER.error(str(error))
         return 1
 
     def _converter(path):

tests/test_execute.py

@@ -1,6 +1,8 @@
 import os
 import pytest
 
+from myst_nb.execution import ExecutionError
+
 
 def regress_nb_doc(file_regression, sphinx_run, check_nbs):
     file_regression.check(
@@ -300,3 +302,27 @@ def test_custom_convert_cache(sphinx_run, file_regression, check_nbs):
     assert "custom-formats" in sphinx_run.env.nb_execution_data
     assert sphinx_run.env.nb_execution_data["custom-formats"]["method"] == "cache"
     assert sphinx_run.env.nb_execution_data["custom-formats"]["succeeded"] is True
+
+
+@pytest.mark.sphinx_params(
+    "basic_failing.ipynb",
+    conf={"execution_allow_errors": False, "execution_fail_on_error": True},
+)
+def test_execution_fail_on_error(sphinx_run, file_regression, check_nbs):
+    with pytest.raises(ExecutionError) as excinfo:
+        sphinx_run.build()
+    assert str(excinfo.value).startswith("Execution failed for file:")
+    # Ensure filename is reported:
+    assert "basic_failing.ipynb" in str(excinfo.value)
+    # Ensure failing code is reported:
+    assert "raise Exception('oopsie!')" in str(excinfo.value)
+
+
+@pytest.mark.sphinx_params(
+    "basic_failing.ipynb",
+    conf={"execution_allow_errors": True, "execution_fail_on_error": True},
+)
+def test_execution_fail_on_error_allow_errors(sphinx_run, file_regression, check_nbs):
+    sphinx_run.build()
+    assert not sphinx_run.warnings()
+    regress_nb_doc(file_regression, sphinx_run, check_nbs)

tests/test_execute/test_execution_fail_on_error_allow_errors.ipynb

@@ -0,0 +1,56 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# a title\n",
+    "\n",
+    "some text\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "Exception",
+     "evalue": "oopsie!",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mException\u001b[0m                                 Traceback (most recent call last)",
+      "\u001b[0;32m<ipython-input-1-714b2b556897>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'oopsie!'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
+      "\u001b[0;31mException\u001b[0m: oopsie!"
+     ]
+    }
+   ],
+   "source": [
+    "raise Exception('oopsie!')"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.2"
+  },
+  "test_name": "notebook1"
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

tests/test_execute/test_execution_fail_on_error_allow_errors.xml

@@ -0,0 +1,12 @@
+<document source="basic_failing">
+    <section ids="a-title" names="a\ title">
+        <title>
+            a title
+        <paragraph>
+            some text
+        <CellNode cell_type="code" classes="cell">
+            <CellInputNode classes="cell_input">
+                <literal_block language="ipython3" xml:space="preserve">
+                    raise Exception('oopsie!')
+            <CellOutputNode classes="cell_output">
+                <CellOutputBundleNode output_count="1">