From 774067ca8001da9f03b5f9422dbc5b2155e882f3 Mon Sep 17 00:00:00 2001 From: Jammy2211 Date: Thu, 30 Jul 2026 11:02:15 +0100 Subject: [PATCH] fix: split a docstring that follows code into its own notebook cell MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `ipynb-py-convert`'s `py2nb` splits cells on the literal "\n\n# %%\n" — the marker must be preceded by a blank line. The docstring-opener branch of `add_notebook_quotes` emitted "# %%\n" after a single newline, so a column-0 docstring opened on the line immediately after code never split: the marker and both delimiters ended up inside the preceding code cell as literal text, making that cell a SyntaxError for anyone who ran it. 13 committed notebook cells across four workspace repos were broken this way. Emit the separator when it is missing, under two constraints. Never when `out` is empty: `py2nb` strips a leading "# %%\n" header, and a leading blank line defeats that strip and yields a spurious empty first code cell. Never when the output already ends blank: emitting it unconditionally would append a trailing blank line to every code cell in every generated notebook. Also raise on a hand-written column-0 `# %%` in a source script. This function is what inserts cell markers, so an authored one collides with the generated one and corrupts the conversion the same way; two workspace scripts carried these for years. Loud beats laundering it into a broken artefact. Tests cover the docstring-after-code split, the stray-marker raise, and — as the pin that makes the workspaces' trailing `"""Finish."""` crutch provably unnecessary — that a script ending in a code cell converts intact. Refs #211 Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_013XzXdQMU3KV2tyZaMNPvsM --- autohands/add_notebook_quotes.py | 30 +++++++++ tests/test_add_notebook_quotes.py | 106 ++++++++++++++++++++++++++++++ 2 files changed, 136 insertions(+) diff --git a/autohands/add_notebook_quotes.py b/autohands/add_notebook_quotes.py index a87e6488..1b875695 100644 --- a/autohands/add_notebook_quotes.py +++ b/autohands/add_notebook_quotes.py @@ -103,6 +103,19 @@ def add_notebook_quotes(lines: Iterable[str]): producing an empty code segment whose duplicate ``# %%`` markers are interpreted as literal code by ``ipynb-py-convert``. + ``ipynb-py-convert``'s ``py2nb`` splits cells on the literal + ``"\n\n# %%\n"`` — the marker must be preceded by a *blank* line. A + docstring opened on the line immediately after code has only a single + newline before it, so the split never fires and the marker plus both + ``'''`` delimiters end up inside the preceding code cell as literal text + (a ``SyntaxError`` for anyone who runs it). The separator is therefore + emitted here when it is missing, subject to two constraints: never when + ``out`` is empty, because ``py2nb`` strips a *leading* ``# %%\n`` header + and a leading blank line would defeat that strip and yield a spurious + empty first code cell; and never when the output already ends blank, + because emitting it unconditionally would append a trailing blank line to + every code cell in every generated notebook. + Used for conversion to ipynb notebooks Parameters @@ -115,6 +128,21 @@ def add_notebook_quotes(lines: Iterable[str]): Lines with %% inserted before and after docs """ lines = strip_env_declarations(list(lines)) + + # A column-0 `# %%` in a *source* script is always a defect: this function + # is what inserts the cell markers, so a hand-written one collides with the + # generated marker and `py2nb` silently folds the following docstring into + # the preceding code cell. Two workspace scripts carried these for years, + # shipping notebooks whose first code cell was a SyntaxError. Fail loudly + # rather than laundering it into a broken artefact. + stray = [n + 1 for n, line in enumerate(lines) if line.rstrip("\r\n") == "# %%"] + if stray: + raise ValueError( + f"source script contains hand-written '# %%' cell marker(s) at " + f"line(s) {stray} — notebook cell markers are generated, not " + f"authored. Delete them; the docstring blocks alone define the cells." + ) + out = list() is_in_quotes = False pending_code_boundary = False @@ -130,6 +158,8 @@ def add_notebook_quotes(lines: Iterable[str]): out.extend(pending_lines) pending_lines = [] pending_code_boundary = False + if out and not "".join(out[-3:]).endswith("\n\n"): + out.append("\n") out.extend(["# %%", "\n", "'''\n"]) is_in_quotes = not is_in_quotes diff --git a/tests/test_add_notebook_quotes.py b/tests/test_add_notebook_quotes.py index a4bd5498..3bc9137a 100644 --- a/tests/test_add_notebook_quotes.py +++ b/tests/test_add_notebook_quotes.py @@ -28,10 +28,58 @@ ) +# A docstring opened on the line *immediately* after code, with no blank line +# between. ``py2nb`` splits on "\n\n# %%\n", so without a separator the marker +# and both delimiters are swallowed into the preceding code cell. +DOCSTRING_AFTER_CODE_SCRIPT = ( + '"""\n' + "__Intro__\n" + '"""\n' + "\n" + "value = 1\n" + '"""\n' + "__Swallowed__\n" + '"""\n' + "\n" + "other = 2\n" +) + + +# A script whose final segment is code, not a docstring. Workspace examples used +# to append a trailing ``"""\nFinish.\n"""`` block in the belief that this shape +# converted badly; it does not, and this test pins that so the crutch cannot be +# reintroduced. +ENDS_WITH_CODE_SCRIPT = ( + '"""\n' + "__Intro__\n" + '"""\n' + "\n" + "first = 1\n" + "\n" + '"""\n' + "__Section__\n" + '"""\n' + "\n" + "last = 2\n" + "print(last)\n" +) + + def _lines(text: str): return text.splitlines(keepends=True) +def _notebook_from(script_text: str, tmp_path, monkeypatch, name): + """Convert *script_text* through the real generation chain.""" + import build_util + + script = tmp_path / name + script.write_text(script_text) + monkeypatch.chdir(tmp_path) + + return json.loads(build_util.py_to_notebook(script).read_text()) + + def test_adjacent_docstrings_do_not_emit_an_empty_code_cell_boundary(): converted = "".join(add_notebook_quotes(_lines(ADJACENT_SCRIPT))) @@ -64,3 +112,61 @@ def test_adjacent_docstrings_generate_separate_markdown_cells(tmp_path, monkeypa source = "".join(cell["source"]) assert "# %%" not in source assert "'''" not in source + + +def test_docstring_immediately_after_code_is_its_own_markdown_cell( + tmp_path, monkeypatch +): + notebook = _notebook_from( + DOCSTRING_AFTER_CODE_SCRIPT, tmp_path, monkeypatch, "after_code.py" + ) + + assert [cell["cell_type"] for cell in notebook["cells"]] == [ + "markdown", + "code", + "markdown", + "code", + ] + assert "__Swallowed__" in "".join(notebook["cells"][2]["source"]) + + for cell in notebook["cells"]: + if cell["cell_type"] == "code": + source = "".join(cell["source"]) + assert "# %%" not in source + assert "'''" not in source + + +def test_script_ending_in_code_keeps_a_complete_final_code_cell( + tmp_path, monkeypatch +): + notebook = _notebook_from( + ENDS_WITH_CODE_SCRIPT, tmp_path, monkeypatch, "ends_with_code.py" + ) + + assert notebook["cells"][-1]["cell_type"] == "code" + + final = "".join(notebook["cells"][-1]["source"]) + assert "last = 2" in final + assert "print(last)" in final + + +def test_hand_written_cell_marker_in_source_raises(): + import pytest + + script = "# %%\n" + ENDS_WITH_CODE_SCRIPT + + with pytest.raises(ValueError, match="hand-written '# %%'"): + add_notebook_quotes(_lines(script)) + + +def test_leading_docstring_does_not_produce_an_empty_first_code_cell( + tmp_path, monkeypatch +): + notebook = _notebook_from( + ENDS_WITH_CODE_SCRIPT, tmp_path, monkeypatch, "leading.py" + ) + + first = notebook["cells"][0] + assert first["cell_type"] == "markdown" + assert "__Intro__" in "".join(first["source"]) + assert [cell["cell_type"] for cell in notebook["cells"]].count("markdown") == 2