Add reStructuredText parsing functions to SphinxDirective (#12492)

CHANGES.rst

@@ -34,6 +34,18 @@ Features added
   Patch by James Addison.
 * #12319: ``sphinx.ext.extlinks``: Add ``extlink-{name}`` CSS class to links.
   Patch by Hugo van Kemenade.
+* Add helper methods for parsing reStructuredText content into nodes from
+  within a directive.
+
+  - :py:meth:`~sphinx.util.docutils.SphinxDirective.parse_content_to_nodes()`
+    parses the directive's content and returns a list of Docutils nodes.
+  - :py:meth:`~sphinx.util.docutils.SphinxDirective.parse_text_to_nodes()`
+    parses the provided text and returns a list of Docutils nodes.
+  - :py:meth:`~sphinx.util.docutils.SphinxDirective.parse_inline()`
+    parses the provided text into inline elements and text nodes.
+
+  Patch by Adam Turner.
+
 
 Bugs fixed
 ----------

doc/conf.py

@@ -186,6 +186,7 @@
     ('py:class', 'NullTranslations'),  # gettext.NullTranslations
     ('py:class', 'RoleFunction'),  # sphinx.domains.Domain
     ('py:class', 'Theme'),  # sphinx.application.TemplateBridge
+    ('py:class', 'system_message'),  # sphinx.utils.docutils
     ('py:class', 'TitleGetter'),  # sphinx.domains.Domain
     ('py:class', 'XRefRole'),  # sphinx.domains.Domain
     ('py:class', 'docutils.nodes.Element'),

doc/development/tutorials/examples/todo.py

@@ -38,7 +38,7 @@ def run(self):
 
         todo_node = todo('\n'.join(self.content))
         todo_node += nodes.title(_('Todo'), _('Todo'))
-        self.state.nested_parse(self.content, self.content_offset, todo_node)
+        todo_node += self.parse_content_to_nodes()
 
         if not hasattr(self.env, 'todo_all_todos'):
             self.env.todo_all_todos = []

sphinx/directives/__init__.py

@@ -13,7 +13,6 @@
 from sphinx.util import docutils
 from sphinx.util.docfields import DocFieldTransformer, Field, TypedField
 from sphinx.util.docutils import SphinxDirective
-from sphinx.util.nodes import nested_parse_with_titles
 from sphinx.util.typing import ExtensionMetadata, OptionSpec  # NoQA: TCH001
 
 if TYPE_CHECKING:
@@ -127,7 +126,7 @@ def before_content(self) -> None:
         """
         pass
 
-    def transform_content(self, contentnode: addnodes.desc_content) -> None:
+    def transform_content(self, content_node: addnodes.desc_content) -> None:
         """
         Called after creating the content through nested parsing,
         but before the ``object-description-transform`` event is emitted,
@@ -275,18 +274,16 @@ def run(self) -> list[Node]:
                     # description of the object with this name in this desc block
                     self.add_target_and_index(name, sig, signode)
 
-        contentnode = addnodes.desc_content()
-        node.append(contentnode)
-
         if self.names:
             # needed for association of version{added,changed} directives
             self.env.temp_data['object'] = self.names[0]
         self.before_content()
-        nested_parse_with_titles(self.state, self.content, contentnode, self.content_offset)
-        self.transform_content(contentnode)
+        content_node = addnodes.desc_content('', *self.parse_content_to_nodes())
+        node.append(content_node)
+        self.transform_content(content_node)
         self.env.app.emit('object-description-transform',
-                          self.domain, self.objtype, contentnode)
-        DocFieldTransformer(self).transform_all(contentnode)
+                          self.domain, self.objtype, content_node)
+        DocFieldTransformer(self).transform_all(content_node)
         self.env.temp_data['object'] = None
         self.after_content()
 

sphinx/directives/code.py

@@ -7,7 +7,6 @@
 
 from docutils import nodes
 from docutils.parsers.rst import directives
-from docutils.statemachine import StringList
 
 from sphinx import addnodes
 from sphinx.directives import optional_int
@@ -75,15 +74,13 @@ def container_wrapper(
 ) -> nodes.container:
     container_node = nodes.container('', literal_block=True,
                                      classes=['literal-block-wrapper'])
-    parsed = nodes.Element()
-    directive.state.nested_parse(StringList([caption], source=''),
-                                 directive.content_offset, parsed)
-    if isinstance(parsed[0], nodes.system_message):
-        msg = __('Invalid caption: %s' % parsed[0].astext())
+    parsed = directive.parse_text_to_nodes(caption, offset=directive.content_offset)
+    node = parsed[0]
+    if isinstance(node, nodes.system_message):
+        msg = __('Invalid caption: %s') % node.astext()
         raise ValueError(msg)
-    if isinstance(parsed[0], nodes.Element):
-        caption_node = nodes.caption(parsed[0].rawsource, '',
-                                     *parsed[0].children)
+    if isinstance(node, nodes.Element):
+        caption_node = nodes.caption(node.rawsource, '', *node.children)
         caption_node.source = literal_node.source
         caption_node.line = literal_node.line
         container_node += caption_node

sphinx/directives/other.py

@@ -198,7 +198,7 @@ def run(self) -> list[Node]:
         else:
             text = _('Author: ')
         emph += nodes.Text(text)
-        inodes, messages = self.state.inline_text(self.arguments[0], self.lineno)
+        inodes, messages = self.parse_inline(self.arguments[0])
         emph.extend(inodes)
 
         ret: list[Node] = [para]
@@ -247,7 +247,7 @@ def run(self) -> list[Node]:
         if not self.arguments:
             return []
         subnode: Element = addnodes.centered()
-        inodes, messages = self.state.inline_text(self.arguments[0], self.lineno)
+        inodes, messages = self.parse_inline(self.arguments[0])
         subnode.extend(inodes)
 
         ret: list[Node] = [subnode]
@@ -267,15 +267,12 @@ class Acks(SphinxDirective):
     option_spec: ClassVar[OptionSpec] = {}
 
     def run(self) -> list[Node]:
-        node = addnodes.acks()
-        node.document = self.state.document
-        self.state.nested_parse(self.content, self.content_offset, node)
-        if len(node.children) != 1 or not isinstance(node.children[0],
-                                                     nodes.bullet_list):
+        children = self.parse_content_to_nodes()
+        if len(children) != 1 or not isinstance(children[0], nodes.bullet_list):
             logger.warning(__('.. acks content is not a list'),
                            location=(self.env.docname, self.lineno))
             return []
-        return [node]
+        return [addnodes.acks('', *children)]
 
 
 class HList(SphinxDirective):
@@ -293,15 +290,12 @@ class HList(SphinxDirective):
 
     def run(self) -> list[Node]:
         ncolumns = self.options.get('columns', 2)
-        node = nodes.paragraph()
-        node.document = self.state.document
-        self.state.nested_parse(self.content, self.content_offset, node)
-        if len(node.children) != 1 or not isinstance(node.children[0],
-                                                     nodes.bullet_list):
+        children = self.parse_content_to_nodes()
+        if len(children) != 1 or not isinstance(children[0], nodes.bullet_list):
             logger.warning(__('.. hlist content is not a list'),
                            location=(self.env.docname, self.lineno))
             return []
-        fulllist = node.children[0]
+        fulllist = children[0]
         # create a hlist node where the items are distributed
         npercol, nmore = divmod(len(fulllist), ncolumns)
         index = 0

sphinx/domains/changeset.py

@@ -62,15 +62,14 @@ def run(self) -> list[Node]:
         node['version'] = self.arguments[0]
         text = versionlabels[self.name] % self.arguments[0]
         if len(self.arguments) == 2:
-            inodes, messages = self.state.inline_text(self.arguments[1],
-                                                      self.lineno + 1)
+            inodes, messages = self.parse_inline(self.arguments[1], lineno=self.lineno + 1)
             para = nodes.paragraph(self.arguments[1], '', *inodes, translatable=False)
             self.set_source_info(para)
             node.append(para)
         else:
             messages = []
         if self.content:
-            self.state.nested_parse(self.content, self.content_offset, node)
+            node += self.parse_content_to_nodes()
         classes = ['versionmodified', versionlabel_classes[self.name]]
         if len(node) > 0 and isinstance(node[0], nodes.paragraph):
             # the contents start with a paragraph

sphinx/domains/cpp/__init__.py

@@ -763,10 +763,9 @@ def run(self) -> list[Node]:
         for sig in signatures:
             node.append(AliasNode(sig, aliasOptions, env=self.env))
 
-        contentnode = addnodes.desc_content()
-        node.append(contentnode)
         self.before_content()
-        self.state.nested_parse(self.content, self.content_offset, contentnode)
+        content_node = addnodes.desc_content('', *self.parse_content_to_nodes())
+        node.append(content_node)
         self.env.temp_data['object'] = None
         self.after_content()
         return [node]

sphinx/domains/javascript.py

@@ -17,7 +17,7 @@
 from sphinx.util import logging
 from sphinx.util.docfields import Field, GroupedField, TypedField
 from sphinx.util.docutils import SphinxDirective
-from sphinx.util.nodes import make_id, make_refnode, nested_parse_with_titles
+from sphinx.util.nodes import make_id, make_refnode
 
 if TYPE_CHECKING:
     from collections.abc import Iterator
@@ -311,10 +311,7 @@ def run(self) -> list[Node]:
         self.env.ref_context['js:module'] = mod_name
         no_index = 'no-index' in self.options or 'noindex' in self.options
 
-        content_node: Element = nodes.section()
-        # necessary so that the child nodes get the right source/line set
-        content_node.document = self.state.document
-        nested_parse_with_titles(self.state, self.content, content_node, self.content_offset)
+        content_nodes = self.parse_content_to_nodes()
 
         ret: list[Node] = []
         if not no_index:
@@ -334,7 +331,7 @@ def run(self) -> list[Node]:
             target = nodes.target('', '', ids=[node_id], ismod=True)
             self.state.document.note_explicit_target(target)
             ret.append(target)
-        ret.extend(content_node.children)
+        ret.extend(content_nodes)
         return ret
 
 

sphinx/domains/python/__init__.py

@@ -22,7 +22,6 @@
     find_pending_xref_condition,
     make_id,
     make_refnode,
-    nested_parse_with_titles,
 )
 
 if TYPE_CHECKING:
@@ -417,10 +416,7 @@ def run(self) -> list[Node]:
         no_index = 'no-index' in self.options or 'noindex' in self.options
         self.env.ref_context['py:module'] = modname
 
-        content_node: Element = nodes.section()
-        # necessary so that the child nodes get the right source/line set
-        content_node.document = self.state.document
-        nested_parse_with_titles(self.state, self.content, content_node, self.content_offset)
+        content_nodes = self.parse_content_to_nodes()
 
         ret: list[Node] = []
         if not no_index:
@@ -444,7 +440,7 @@ def run(self) -> list[Node]:
             # The node order is: index node first, then target node.
             ret.append(inode)
             ret.append(target)
-        ret.extend(content_node.children)
+        ret.extend(content_nodes)
         return ret
 
 

sphinx/domains/std/__init__.py

@@ -20,6 +20,7 @@
 from sphinx.util import docname_join, logging, ws_re
 from sphinx.util.docutils import SphinxDirective
 from sphinx.util.nodes import clean_astext, make_id, make_refnode
+from sphinx.util.parsing import nested_parse_to_nodes
 
 if TYPE_CHECKING:
     from collections.abc import Iterable, Iterator
@@ -260,10 +261,15 @@ def process_link(self, env: BuildEnvironment, refnode: Element, has_explicit_tit
         return title, target
 
 
-def split_term_classifiers(line: str) -> list[str | None]:
+_term_classifiers_re = re.compile(' +: +')
+
+
+def split_term_classifiers(line: str) -> tuple[str, str | None]:
     # split line into a term and classifiers. if no classifier, None is used..
-    parts: list[str | None] = [*re.split(' +: +', line), None]
-    return parts
+    parts = _term_classifiers_re.split(line)
+    term = parts[0]
+    first_classifier = parts[1] if len(parts) >= 2 else None
+    return term, first_classifier
 
 
 def make_glossary_term(env: BuildEnvironment, textnodes: Iterable[Node], index_key: str,
@@ -382,27 +388,27 @@ def run(self) -> list[Node]:
             termnodes: list[Node] = []
             system_messages: list[Node] = []
             for line, source, lineno in terms:
-                parts = split_term_classifiers(line)
+                term_, first_classifier = split_term_classifiers(line)
                 # parse the term with inline markup
                 # classifiers (parts[1:]) will not be shown on doctree
-                textnodes, sysmsg = self.state.inline_text(parts[0],
-                                                           lineno)
+                textnodes, sysmsg = self.parse_inline(term_, lineno=lineno)
 
                 # use first classifier as a index key
                 term = make_glossary_term(self.env, textnodes,
-                                          parts[1], source, lineno,  # type: ignore[arg-type]
+                                          first_classifier, source, lineno,  # type: ignore[arg-type]
                                           node_id=None, document=self.state.document)
                 term.rawsource = line
                 system_messages.extend(sysmsg)
                 termnodes.append(term)
 
             termnodes.extend(system_messages)
 
-            defnode = nodes.definition()
             if definition:
-                self.state.nested_parse(definition, definition.items[0][1],
-                                        defnode)
-            termnodes.append(defnode)
+                offset = definition.items[0][1]
+                definition_nodes = nested_parse_to_nodes(self.state, definition, offset=offset)
+            else:
+                definition_nodes = []
+            termnodes.append(nodes.definition('', *definition_nodes))
             items.append(nodes.definition_list_item('', *termnodes))
 
         dlist = nodes.definition_list('', *items)

sphinx/ext/autodoc/directive.py

@@ -9,10 +9,10 @@
 from sphinx.ext.autodoc import Documenter, Options
 from sphinx.util import logging
 from sphinx.util.docutils import SphinxDirective, switch_source_input
-from sphinx.util.nodes import nested_parse_with_titles
+from sphinx.util.parsing import nested_parse_to_nodes
 
 if TYPE_CHECKING:
-    from docutils.nodes import Element, Node
+    from docutils.nodes import Node
     from docutils.parsers.rst.states import RSTState
 
     from sphinx.config import Config
@@ -86,15 +86,12 @@ def parse_generated_content(state: RSTState, content: StringList, documenter: Do
     """Parse an item of content generated by Documenter."""
     with switch_source_input(state, content):
         if documenter.titles_allowed:
-            node: Element = nodes.section()
-            # necessary so that the child nodes get the right source/line set
-            node.document = state.document
-            nested_parse_with_titles(state, content, node)
-        else:
-            node = nodes.paragraph()
-            node.document = state.document
-            state.nested_parse(content, 0, node)
+            return nested_parse_to_nodes(state, content)
 
+        node = nodes.paragraph()
+        # necessary so that the child nodes get the right source/line set
+        node.document = state.document
+        state.nested_parse(content, 0, node, match_titles=False)
         return node.children
 
 

sphinx/ext/autosummary/__init__.py

@@ -87,6 +87,7 @@
 )
 from sphinx.util.inspect import getmro, signature_from_str
 from sphinx.util.matching import Matcher
+from sphinx.util.parsing import nested_parse_to_nodes
 
 if TYPE_CHECKING:
     from collections.abc import Sequence
@@ -406,16 +407,13 @@ def append_row(*column_texts: str) -> None:
             row = nodes.row('')
             source, line = self.state_machine.get_source_and_line()
             for text in column_texts:
-                node = nodes.paragraph('')
-                vl = StringList()
-                vl.append(text, '%s:%d:<autosummary>' % (source, line))
+                vl = StringList([text], f'{source}:{line}:<autosummary>')
                 with switch_source_input(self.state, vl):
-                    self.state.nested_parse(vl, 0, node)
-                    try:
-                        if isinstance(node[0], nodes.paragraph):
-                            node = node[0]
-                    except IndexError:
-                        pass
+                    col_nodes = nested_parse_to_nodes(self.state, vl)
+                    if col_nodes and isinstance(col_nodes[0], nodes.paragraph):
+                        node = col_nodes[0]
+                    else:
+                        node = nodes.paragraph('')
                     row.append(nodes.entry('', node))
             body.append(row)