From d318a49f532cd5a43f22bb3105f61e5c6ada3862 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 17 Jan 2024 22:38:14 -0800 Subject: [PATCH 01/10] src/sage_docbuild/conf.py: Show preparsed doctests (and Jupyter cells) using inline tabs --- src/sage_docbuild/conf.py | 101 ++++++++++++++++++++++++++++---------- 1 file changed, 76 insertions(+), 25 deletions(-) diff --git a/src/sage_docbuild/conf.py b/src/sage_docbuild/conf.py index 6b84e772dd1..8d7d9e470da 100644 --- a/src/sage_docbuild/conf.py +++ b/src/sage_docbuild/conf.py @@ -230,7 +230,7 @@ def sphinx_plot(graphics, **kwds): # console lexers. 'ipycon' is the IPython console, which is what we want # for most code blocks: anything with "sage:" prompts. For other IPython, # like blocks which might appear in a notebook cell, use 'ipython'. -highlighting.lexers['ipycon'] = IPythonConsoleLexer(in1_regex=r'sage: ', in2_regex=r'[.][.][.][.]: ') +highlighting.lexers['ipycon'] = IPythonConsoleLexer(in1_regex=r'(sage:|>>>)', in2_regex=r'([.][.][.][.]:|[.][.][.])') highlighting.lexers['ipython'] = IPyLexer() highlight_language = 'ipycon' @@ -790,6 +790,9 @@ class will be properly documented inside its surrounding class. from jupyter_sphinx.ast import JupyterCellNode, CellInputNode +from docutils.nodes import container as Container, label as Label, literal_block as LiteralBlock, Text +from sphinx_inline_tabs._impl import TabContainer +from sage.repl.preparse import preparse class SagecodeTransform(SphinxTransform): """ @@ -828,29 +831,78 @@ def apply(self): if self.app.builder.tags.has('html') or self.app.builder.tags.has('inventory'): for node in self.document.traverse(nodes.literal_block): if node.get('language') is None and node.astext().startswith('sage:'): - source = node.rawsource - lines = [] - for line in source.splitlines(): + parent = node.parent + index = parent.index(node) + parent.remove(node) + # Tab for Sage code + container = TabContainer("", type="tab", new_set=False) + textnodes = [Text('Sage')] + label = Label("", "", *textnodes) + container += label + content = Container("", is_div=True, classes=["tab-content"]) + content += node + container += content + parent.insert(index, container) + # Tab for preparsed version + container = TabContainer("", type="tab", new_set=False) + textnodes = [Text('Python')] + label = Label("", "", *textnodes) + container += label + content = Container("", is_div=True, classes=["tab-content"]) + example_lines = [] + preparsed_lines = ['>>> from sage.all import *'] + for line in node.rawsource.splitlines() + ['']: # one extra to process last example newline = line.lstrip() - if newline.startswith('sage: ') or newline.startswith('....: '): - lines.append(newline[6:]) - cell_node = JupyterCellNode( - execute=False, - hide_code=True, - hide_output=True, - emphasize_lines=[], - raises=False, - stderr=True, - code_below=False, - classes=["jupyter_cell"]) - cell_input = CellInputNode(classes=['cell_input','live-doc']) - cell_input += nodes.literal_block( - text='\n'.join(lines), - linenos=False, - linenostart=1) - cell_node += cell_input - - node.parent.insert(node.parent.index(node) + 1, cell_node) + if newline.startswith('....: '): + example_lines.append(newline[6:]) + else: + if example_lines: + preparsed_example = preparse('\n'.join(example_lines)) + prompt = '>>> ' + for preparsed_line in preparsed_example.splitlines(): + preparsed_lines.append(prompt + preparsed_line) + prompt = '... ' + example_lines = [] + if newline.startswith('sage: '): + example_lines.append(newline[6:]) + else: + preparsed_lines.append(line) + preparsed = '\n'.join(preparsed_lines) + preparsed_node = LiteralBlock(preparsed, preparsed, language='ipycon') + content += preparsed_node + container += content + parent.insert(index + 1, container) + if os.environ.get('SAGE_LIVE_DOC', 'no') == 'yes': + # Tab for Jupyter-sphinx cell + source = node.rawsource + lines = [] + for line in source.splitlines(): + newline = line.lstrip() + if newline.startswith('sage: ') or newline.startswith('....: '): + lines.append(newline[6:]) + cell_node = JupyterCellNode( + execute=False, + hide_code=False, + hide_output=True, + emphasize_lines=[], + raises=False, + stderr=True, + code_below=False, + classes=["jupyter_cell"]) + cell_input = CellInputNode(classes=['cell_input','live-doc']) + cell_input += nodes.literal_block( + text='\n'.join(lines), + linenos=False, + linenostart=1) + cell_node += cell_input + container = TabContainer("", type="tab", new_set=False) + textnodes = [Text('Sage (live)')] + label = Label("", "", *textnodes) + container += label + content = Container("", is_div=True, classes=["tab-content"]) + content += cell_node + container += content + parent.insert(index + 1, container) # This replaces the setup() in sage.misc.sagedoc_conf def setup(app): @@ -863,8 +915,7 @@ def setup(app): app.connect('autodoc-process-docstring', skip_TESTS_block) app.connect('autodoc-skip-member', skip_member) app.add_transform(SagemathTransform) - if os.environ.get('SAGE_LIVE_DOC', 'no') == 'yes': - app.add_transform(SagecodeTransform) + app.add_transform(SagecodeTransform) # When building the standard docs, app.srcdir is set to SAGE_DOC_SRC + # 'LANGUAGE/DOCNAME'. From 10acc0f34c328d89bd255ab97018bb9c19c400e8 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 17 Jan 2024 23:22:45 -0800 Subject: [PATCH 02/10] src/sage_docbuild/conf.py: Include plain Python prompts in copybutton config --- src/sage_docbuild/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sage_docbuild/conf.py b/src/sage_docbuild/conf.py index 8d7d9e470da..b998e02b6b9 100644 --- a/src/sage_docbuild/conf.py +++ b/src/sage_docbuild/conf.py @@ -305,7 +305,7 @@ def set_intersphinx_mappings(app, config): multidocs_is_master = True # https://sphinx-copybutton.readthedocs.io/en/latest/use.html -copybutton_prompt_text = r"sage: |[.][.][.][.]: |\$ " +copybutton_prompt_text = r"sage: |[.][.][.][.]: |>>> |[.][.][.] |\$ " copybutton_prompt_is_regexp = True copybutton_exclude = '.linenos, .c1' # exclude single comments (in particular, # optional!) copybutton_only_copy_prompt_lines = True From 8974bcb94e5686547f7ca2160004cc70f65ef390 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 18 Jan 2024 09:52:52 -0800 Subject: [PATCH 03/10] src/sage_docbuild/conf.py: Conditionalize on environment variable SAGE_PREPARSED_DOC (default yes) --- src/sage_docbuild/conf.py | 74 +++++++++++++++++++++------------------ 1 file changed, 39 insertions(+), 35 deletions(-) diff --git a/src/sage_docbuild/conf.py b/src/sage_docbuild/conf.py index b998e02b6b9..a4bc5955e90 100644 --- a/src/sage_docbuild/conf.py +++ b/src/sage_docbuild/conf.py @@ -39,6 +39,9 @@ # General configuration # --------------------- +SAGE_LIVE_DOC = os.environ.get('SAGE_LIVE_DOC', 'no') +SAGE_PREPARSED_DOC = os.environ.get('SAGE_PREPARSED_DOC', 'yes') + # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [ @@ -57,7 +60,7 @@ jupyter_execute_default_kernel = 'sagemath' -if os.environ.get('SAGE_LIVE_DOC', 'no') == 'yes': +if SAGE_LIVE_DOC == 'yes': SAGE_JUPYTER_SERVER = os.environ.get('SAGE_JUPYTER_SERVER', 'binder') if SAGE_JUPYTER_SERVER.startswith('binder'): # format: "binder" or @@ -789,11 +792,6 @@ class will be properly documented inside its surrounding class. return skip -from jupyter_sphinx.ast import JupyterCellNode, CellInputNode -from docutils.nodes import container as Container, label as Label, literal_block as LiteralBlock, Text -from sphinx_inline_tabs._impl import TabContainer -from sage.repl.preparse import preparse - class SagecodeTransform(SphinxTransform): """ Transform a code block to a live code block enabled by jupyter-sphinx. @@ -831,6 +829,8 @@ def apply(self): if self.app.builder.tags.has('html') or self.app.builder.tags.has('inventory'): for node in self.document.traverse(nodes.literal_block): if node.get('language') is None and node.astext().startswith('sage:'): + from docutils.nodes import container as Container, label as Label, literal_block as LiteralBlock, Text + from sphinx_inline_tabs._impl import TabContainer parent = node.parent index = parent.index(node) parent.remove(node) @@ -843,37 +843,40 @@ def apply(self): content += node container += content parent.insert(index, container) - # Tab for preparsed version - container = TabContainer("", type="tab", new_set=False) - textnodes = [Text('Python')] - label = Label("", "", *textnodes) - container += label - content = Container("", is_div=True, classes=["tab-content"]) - example_lines = [] - preparsed_lines = ['>>> from sage.all import *'] - for line in node.rawsource.splitlines() + ['']: # one extra to process last example - newline = line.lstrip() - if newline.startswith('....: '): - example_lines.append(newline[6:]) - else: - if example_lines: - preparsed_example = preparse('\n'.join(example_lines)) - prompt = '>>> ' - for preparsed_line in preparsed_example.splitlines(): - preparsed_lines.append(prompt + preparsed_line) - prompt = '... ' - example_lines = [] - if newline.startswith('sage: '): + if SAGE_PREPARSED_DOC == 'yes': + # Tab for preparsed version + from sage.repl.preparse import preparse + container = TabContainer("", type="tab", new_set=False) + textnodes = [Text('Python')] + label = Label("", "", *textnodes) + container += label + content = Container("", is_div=True, classes=["tab-content"]) + example_lines = [] + preparsed_lines = ['>>> from sage.all import *'] + for line in node.rawsource.splitlines() + ['']: # one extra to process last example + newline = line.lstrip() + if newline.startswith('....: '): example_lines.append(newline[6:]) else: - preparsed_lines.append(line) - preparsed = '\n'.join(preparsed_lines) - preparsed_node = LiteralBlock(preparsed, preparsed, language='ipycon') - content += preparsed_node - container += content - parent.insert(index + 1, container) - if os.environ.get('SAGE_LIVE_DOC', 'no') == 'yes': + if example_lines: + preparsed_example = preparse('\n'.join(example_lines)) + prompt = '>>> ' + for preparsed_line in preparsed_example.splitlines(): + preparsed_lines.append(prompt + preparsed_line) + prompt = '... ' + example_lines = [] + if newline.startswith('sage: '): + example_lines.append(newline[6:]) + else: + preparsed_lines.append(line) + preparsed = '\n'.join(preparsed_lines) + preparsed_node = LiteralBlock(preparsed, preparsed, language='ipycon') + content += preparsed_node + container += content + parent.insert(index + 1, container) + if SAGE_LIVE_DOC == 'yes': # Tab for Jupyter-sphinx cell + from jupyter_sphinx.ast import JupyterCellNode, CellInputNode source = node.rawsource lines = [] for line in source.splitlines(): @@ -915,7 +918,8 @@ def setup(app): app.connect('autodoc-process-docstring', skip_TESTS_block) app.connect('autodoc-skip-member', skip_member) app.add_transform(SagemathTransform) - app.add_transform(SagecodeTransform) + if SAGE_LIVE_DOC == 'yes' or SAGE_PREPARSED_DOC == 'yes': + app.add_transform(SagecodeTransform) # When building the standard docs, app.srcdir is set to SAGE_DOC_SRC + # 'LANGUAGE/DOCNAME'. From 6412ce4a5e3eada85555d76d231a886f6e8c83ec Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 18 Jan 2024 10:20:41 -0800 Subject: [PATCH 04/10] sage --docbuild: Add option --no-preparsed-examples --- src/sage_docbuild/__main__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/sage_docbuild/__main__.py b/src/sage_docbuild/__main__.py index 0d15808a69c..15eaf6c5e19 100644 --- a/src/sage_docbuild/__main__.py +++ b/src/sage_docbuild/__main__.py @@ -288,6 +288,9 @@ def setup_parser(): standard.add_argument("--no-plot", dest="no_plot", action="store_true", help="do not include graphics auto-generated using the '.. plot' markup") + standard.add_argument("--no-preparsed-examples", dest="no_preparsed_examples", + action="store_true", + help="do not show preparsed versions of EXAMPLES blocks") standard.add_argument("--include-tests-blocks", dest="skip_tests", default=True, action="store_false", help="include TESTS blocks in the reference manual") @@ -477,6 +480,8 @@ def excepthook(*exc_info): build_options.ALLSPHINXOPTS += "-n " if args.no_plot: os.environ['SAGE_SKIP_PLOT_DIRECTIVE'] = 'yes' + if args.no_preparsed_examples: + os.environ['SAGE_PREPARSED_DOC'] = 'no' if args.live_doc: os.environ['SAGE_LIVE_DOC'] = 'yes' if args.skip_tests: From 161a7e761b740a50f1c47cdca8098594b12706ee Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 18 Jan 2024 10:28:13 -0800 Subject: [PATCH 05/10] src/doc/en/installation/source.rst (SAGE_DOCBUILD_OPTS): Mention --no-preparsed-examples --- src/doc/en/installation/source.rst | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/doc/en/installation/source.rst b/src/doc/en/installation/source.rst index 26454e4d6d3..fbe0d15ac2f 100644 --- a/src/doc/en/installation/source.rst +++ b/src/doc/en/installation/source.rst @@ -940,11 +940,19 @@ Environment variables controlling the documentation build The value of this variable is passed as an argument to ``sage --docbuild all html`` or ``sage --docbuild all pdf`` when - you run ``make``, ``make doc``, or ``make doc-pdf``. For example, you can - add ``--no-plot`` to this variable to avoid building the graphics coming from - the ``.. PLOT`` directive within the documentation, or you can add - ``--include-tests-blocks`` to include all "TESTS" blocks in the reference - manual. Run ``sage --docbuild help`` to see the full list of options. + you run ``make``, ``make doc``, or ``make doc-pdf``. For example: + + - add ``--no-plot`` to this variable to avoid building the graphics coming from + the ``.. PLOT`` directive within the documentation, + + - add ``--no-preparsed-examples`` to only show the original Sage code of + "EXAMPLES" blocks, suppressing the tab with the preparsed, plain Python + version, or + + - add ``--include-tests-blocks`` to include all "TESTS" blocks in the reference + manual. + + Run ``sage --docbuild help`` to see the full list of options. .. envvar:: SAGE_SPKG_INSTALL_DOCS From 0034cc9b5d3ecb816b731560d96ba9580e3da91d Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 18 Jan 2024 12:17:14 -0800 Subject: [PATCH 06/10] .github/workflows/[doc-]build[-pdf].yml: Do not fail when there are no blockers --- .github/workflows/build.yml | 4 ++-- .github/workflows/doc-build-pdf.yml | 4 ++-- .github/workflows/doc-build.yml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 56f231d9578..9129874ab55 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -35,13 +35,13 @@ jobs: uses: actions/checkout@v4 - name: Merge CI fixes from sagemath/sage run: | - .ci/merge-fixes.sh + mkdir -p upstream + .ci/merge-fixes.sh 2>&1 | tee upstream/ci_fixes.log env: GH_TOKEN: ${{ github.token }} SAGE_CI_FIXES_FROM_REPOSITORIES: ${{ vars.SAGE_CI_FIXES_FROM_REPOSITORIES }} - name: Store CI fixes in upstream artifact run: | - mkdir -p upstream if git format-patch --stdout test_base > ci_fixes.patch; then cp ci_fixes.patch upstream/ fi diff --git a/.github/workflows/doc-build-pdf.yml b/.github/workflows/doc-build-pdf.yml index 7ae675d9e64..6e6e8776062 100644 --- a/.github/workflows/doc-build-pdf.yml +++ b/.github/workflows/doc-build-pdf.yml @@ -29,13 +29,13 @@ jobs: uses: actions/checkout@v4 - name: Merge CI fixes from sagemath/sage run: | - .ci/merge-fixes.sh + mkdir -p upstream + .ci/merge-fixes.sh 2>&1 | tee upstream/ci_fixes.log env: GH_TOKEN: ${{ github.token }} SAGE_CI_FIXES_FROM_REPOSITORIES: ${{ vars.SAGE_CI_FIXES_FROM_REPOSITORIES }} - name: Store CI fixes in upstream artifact run: | - mkdir -p upstream if git format-patch --stdout test_base > ci_fixes.patch; then cp ci_fixes.patch upstream/ fi diff --git a/.github/workflows/doc-build.yml b/.github/workflows/doc-build.yml index 9d82909ef5f..21e19e3859a 100644 --- a/.github/workflows/doc-build.yml +++ b/.github/workflows/doc-build.yml @@ -24,13 +24,13 @@ jobs: uses: actions/checkout@v4 - name: Merge CI fixes from sagemath/sage run: | - .ci/merge-fixes.sh + mkdir -p upstream + .ci/merge-fixes.sh 2>&1 | tee upstream/ci_fixes.log env: GH_TOKEN: ${{ github.token }} SAGE_CI_FIXES_FROM_REPOSITORIES: ${{ vars.SAGE_CI_FIXES_FROM_REPOSITORIES }} - name: Store CI fixes in upstream artifact run: | - mkdir -p upstream if git format-patch --stdout test_base > ci_fixes.patch; then cp ci_fixes.patch upstream/ fi From d8b0a129ffec9f07f47bc84f3375685f9daa26f7 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 18 Jan 2024 12:18:29 -0800 Subject: [PATCH 07/10] .ci/merge-fixes.sh: Cosmetic change --- .ci/merge-fixes.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.ci/merge-fixes.sh b/.ci/merge-fixes.sh index 73b4c665caf..13350018221 100755 --- a/.ci/merge-fixes.sh +++ b/.ci/merge-fixes.sh @@ -42,6 +42,7 @@ for REPO in ${SAGE_CI_FIXES_FROM_REPOSITORIES:-sagemath/sage}; do # Considered alternative: Use https://github.com/$REPO/pull/$a.diff, # which squashes everything into one diff without commit metadata. PULL_URL="https://github.com/$REPO/pull/$a" + PULL_SHORT="$REPO#$a" PULL_FILE="$REPO_FILE-$a" PATH=build/bin:$PATH build/bin/sage-download-file --quiet "$PULL_URL.patch" $PULL_FILE.patch date -u +"%Y-%m-%dT%H:%M:%SZ" > $PULL_FILE.date # Record the date, for future reference @@ -67,7 +68,7 @@ for REPO in ${SAGE_CI_FIXES_FROM_REPOSITORIES:-sagemath/sage}; do git am --signoff --show-current-patch=diff echo "--------------------------------------------------------------------8<-----------------------------" echo "::endgroup::" - echo "Failure applying $PULL_URL as a patch, resetting" + echo "Failure applying $PULL_SHORT as a patch, resetting" git am --signoff --abort fi done From 84e7c89909c60659b226f6b50d2591fbc53d07b0 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 18 Jan 2024 20:20:50 -0800 Subject: [PATCH 08/10] Activate Thebe when "Sage (live)" tab is clicked --- src/doc/common/static/jupyter-sphinx-furo.js | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/doc/common/static/jupyter-sphinx-furo.js b/src/doc/common/static/jupyter-sphinx-furo.js index 5194ff470fc..01839aa4e99 100644 --- a/src/doc/common/static/jupyter-sphinx-furo.js +++ b/src/doc/common/static/jupyter-sphinx-furo.js @@ -112,3 +112,16 @@ thebelab.on("status", function (evt, data) { kernel.requestExecute({code: "%display latex"}); } }); + +// Activate Thebe when "Sage (live)" tab is clicked +document.querySelectorAll('input[class="tab-input"]').forEach((elem) => { + elem.addEventListener("click", function(event) { + if (elem.nextElementSibling) { + if (elem.nextElementSibling.nextElementSibling) { + if (elem.nextElementSibling.nextElementSibling.querySelector('div[class="thebelab-code"]')) { + initThebelab(); + } + } + } + }); +}); From 57fd4ab7a9ee868188484b9c23c83dbef4f569fd Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Tue, 6 Feb 2024 21:49:56 -0800 Subject: [PATCH 09/10] src/sage_docbuild/conf.py: Do not merge inline tabs of adjacent literal blocks --- src/sage_docbuild/conf.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/sage_docbuild/conf.py b/src/sage_docbuild/conf.py index a4bc5955e90..d1416efc8ba 100644 --- a/src/sage_docbuild/conf.py +++ b/src/sage_docbuild/conf.py @@ -833,6 +833,10 @@ def apply(self): from sphinx_inline_tabs._impl import TabContainer parent = node.parent index = parent.index(node) + if isinstance(node.previous_sibling(), TabContainer): + # Make sure not to merge inline tabs for adjacent literal blocks + parent.insert(index, Text('')) + index += 1 parent.remove(node) # Tab for Sage code container = TabContainer("", type="tab", new_set=False) From dea37e7320f3f129a6545a7821e1139b2a71e34c Mon Sep 17 00:00:00 2001 From: Kwankyu Lee Date: Wed, 20 Mar 2024 09:33:54 -0700 Subject: [PATCH 10/10] src/doc/common/static/custom-jupyter-sphinx.css, src/sage_docbuild/conf.py: Refinements --- src/doc/common/static/custom-jupyter-sphinx.css | 2 +- src/sage_docbuild/conf.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/doc/common/static/custom-jupyter-sphinx.css b/src/doc/common/static/custom-jupyter-sphinx.css index a68a5cb05aa..e79b47b4539 100644 --- a/src/doc/common/static/custom-jupyter-sphinx.css +++ b/src/doc/common/static/custom-jupyter-sphinx.css @@ -1,5 +1,5 @@ div.jupyter_container { - margin: .5rem 0; + border: 0; } div.jupyter_container + div.jupyter_container { diff --git a/src/sage_docbuild/conf.py b/src/sage_docbuild/conf.py index 145105bb814..1c705c68d1f 100644 --- a/src/sage_docbuild/conf.py +++ b/src/sage_docbuild/conf.py @@ -903,7 +903,7 @@ def apply(self): linenostart=1) cell_node += cell_input container = TabContainer("", type="tab", new_set=False) - textnodes = [Text('Sage (live)')] + textnodes = [Text('Sage Live')] label = Label("", "", *textnodes) container += label content = Container("", is_div=True, classes=["tab-content"])