fix: split a docstring that follows code into its own notebook cell

autohands/add_notebook_quotes.py

@@ -103,6 +103,19 @@ def add_notebook_quotes(lines: Iterable[str]):
     producing an empty code segment whose duplicate ``# %%`` markers are
     interpreted as literal code by ``ipynb-py-convert``.
 
+    ``ipynb-py-convert``'s ``py2nb`` splits cells on the literal
+    ``"\n\n# %%\n"`` — the marker must be preceded by a *blank* line. A
+    docstring opened on the line immediately after code has only a single
+    newline before it, so the split never fires and the marker plus both
+    ``'''`` delimiters end up inside the preceding code cell as literal text
+    (a ``SyntaxError`` for anyone who runs it). The separator is therefore
+    emitted here when it is missing, subject to two constraints: never when
+    ``out`` is empty, because ``py2nb`` strips a *leading* ``# %%\n`` header
+    and a leading blank line would defeat that strip and yield a spurious
+    empty first code cell; and never when the output already ends blank,
+    because emitting it unconditionally would append a trailing blank line to
+    every code cell in every generated notebook.
+
     Used for conversion to ipynb notebooks
 
     Parameters
@@ -115,6 +128,21 @@ def add_notebook_quotes(lines: Iterable[str]):
     Lines with %% inserted before and after docs
     """
     lines = strip_env_declarations(list(lines))
+
+    # A column-0 `# %%` in a *source* script is always a defect: this function
+    # is what inserts the cell markers, so a hand-written one collides with the
+    # generated marker and `py2nb` silently folds the following docstring into
+    # the preceding code cell. Two workspace scripts carried these for years,
+    # shipping notebooks whose first code cell was a SyntaxError. Fail loudly
+    # rather than laundering it into a broken artefact.
+    stray = [n + 1 for n, line in enumerate(lines) if line.rstrip("\r\n") == "# %%"]
+    if stray:
+        raise ValueError(
+            f"source script contains hand-written '# %%' cell marker(s) at "
+            f"line(s) {stray} — notebook cell markers are generated, not "
+            f"authored. Delete them; the docstring blocks alone define the cells."
+        )
+
     out = list()
     is_in_quotes = False
     pending_code_boundary = False
@@ -130,6 +158,8 @@ def add_notebook_quotes(lines: Iterable[str]):
                     out.extend(pending_lines)
                     pending_lines = []
                     pending_code_boundary = False
+                if out and not "".join(out[-3:]).endswith("\n\n"):
+                    out.append("\n")
                 out.extend(["# %%", "\n", "'''\n"])
 
             is_in_quotes = not is_in_quotes

tests/test_add_notebook_quotes.py

@@ -28,10 +28,58 @@
 )
 
 
+# A docstring opened on the line *immediately* after code, with no blank line
+# between. ``py2nb`` splits on "\n\n# %%\n", so without a separator the marker
+# and both delimiters are swallowed into the preceding code cell.
+DOCSTRING_AFTER_CODE_SCRIPT = (
+    '"""\n'
+    "__Intro__\n"
+    '"""\n'
+    "\n"
+    "value = 1\n"
+    '"""\n'
+    "__Swallowed__\n"
+    '"""\n'
+    "\n"
+    "other = 2\n"
+)
+
+
+# A script whose final segment is code, not a docstring. Workspace examples used
+# to append a trailing ``"""\nFinish.\n"""`` block in the belief that this shape
+# converted badly; it does not, and this test pins that so the crutch cannot be
+# reintroduced.
+ENDS_WITH_CODE_SCRIPT = (
+    '"""\n'
+    "__Intro__\n"
+    '"""\n'
+    "\n"
+    "first = 1\n"
+    "\n"
+    '"""\n'
+    "__Section__\n"
+    '"""\n'
+    "\n"
+    "last = 2\n"
+    "print(last)\n"
+)
+
+
 def _lines(text: str):
     return text.splitlines(keepends=True)
 
 
+def _notebook_from(script_text: str, tmp_path, monkeypatch, name):
+    """Convert *script_text* through the real generation chain."""
+    import build_util
+
+    script = tmp_path / name
+    script.write_text(script_text)
+    monkeypatch.chdir(tmp_path)
+
+    return json.loads(build_util.py_to_notebook(script).read_text())
+
+
 def test_adjacent_docstrings_do_not_emit_an_empty_code_cell_boundary():
     converted = "".join(add_notebook_quotes(_lines(ADJACENT_SCRIPT)))
 
@@ -64,3 +112,61 @@ def test_adjacent_docstrings_generate_separate_markdown_cells(tmp_path, monkeypa
             source = "".join(cell["source"])
             assert "# %%" not in source
             assert "'''" not in source
+
+
+def test_docstring_immediately_after_code_is_its_own_markdown_cell(
+    tmp_path, monkeypatch
+):
+    notebook = _notebook_from(
+        DOCSTRING_AFTER_CODE_SCRIPT, tmp_path, monkeypatch, "after_code.py"
+    )
+
+    assert [cell["cell_type"] for cell in notebook["cells"]] == [
+        "markdown",
+        "code",
+        "markdown",
+        "code",
+    ]
+    assert "__Swallowed__" in "".join(notebook["cells"][2]["source"])
+
+    for cell in notebook["cells"]:
+        if cell["cell_type"] == "code":
+            source = "".join(cell["source"])
+            assert "# %%" not in source
+            assert "'''" not in source
+
+
+def test_script_ending_in_code_keeps_a_complete_final_code_cell(
+    tmp_path, monkeypatch
+):
+    notebook = _notebook_from(
+        ENDS_WITH_CODE_SCRIPT, tmp_path, monkeypatch, "ends_with_code.py"
+    )
+
+    assert notebook["cells"][-1]["cell_type"] == "code"
+
+    final = "".join(notebook["cells"][-1]["source"])
+    assert "last = 2" in final
+    assert "print(last)" in final
+
+
+def test_hand_written_cell_marker_in_source_raises():
+    import pytest
+
+    script = "# %%\n" + ENDS_WITH_CODE_SCRIPT
+
+    with pytest.raises(ValueError, match="hand-written '# %%'"):
+        add_notebook_quotes(_lines(script))
+
+
+def test_leading_docstring_does_not_produce_an_empty_first_code_cell(
+    tmp_path, monkeypatch
+):
+    notebook = _notebook_from(
+        ENDS_WITH_CODE_SCRIPT, tmp_path, monkeypatch, "leading.py"
+    )
+
+    first = notebook["cells"][0]
+    assert first["cell_type"] == "markdown"
+    assert "__Intro__" in "".join(first["source"])
+    assert [cell["cell_type"] for cell in notebook["cells"]].count("markdown") == 2