nodes.py

"""Docutils node-related utility functions for Sphinx."""

from __future__ import annotations

import contextlib
import re
import unicodedata
from typing import TYPE_CHECKING, Any, Callable, Generic, TypeVar, cast

from docutils import nodes
from docutils.nodes import Node

from sphinx import addnodes
from sphinx.locale import __
from sphinx.util import logging

if TYPE_CHECKING:
    from collections.abc import Iterable, Iterator

    from docutils.nodes import Element
    from docutils.parsers.rst import Directive
    from docutils.parsers.rst.states import Inliner
    from docutils.statemachine import StringList

    from sphinx.builders import Builder
    from sphinx.environment import BuildEnvironment
    from sphinx.util.tags import Tags

logger = logging.getLogger(__name__)


# \x00 means the "<" was backslash-escaped
explicit_title_re = re.compile(r'^(.+?)\s*(?<!\x00)<([^<]*?)>$', re.DOTALL)
caption_ref_re = explicit_title_re  # b/w compat alias


N = TypeVar("N", bound=Node)


class NodeMatcher(Generic[N]):
    """A helper class for Node.findall().

    It checks that the given node is an instance of the specified node-classes and
    has the specified node-attributes.

    For example, following example searches ``reference`` node having ``refdomain``
    and ``reftype`` attributes::

        matcher = NodeMatcher(nodes.reference, refdomain='std', reftype='citation')
        matcher.findall(doctree)
        # => [<reference ...>, <reference ...>, ...]

    A special value ``typing.Any`` matches any kind of node-attributes.  For example,
    following example searches ``reference`` node having ``refdomain`` attributes::

        matcher = NodeMatcher(nodes.reference, refdomain=Any)
        matcher.findall(doctree)
        # => [<reference ...>, <reference ...>, ...]
    """

    def __init__(self, *node_classes: type[N], **attrs: Any) -> None:
        self.classes = node_classes
        self.attrs = attrs

    def match(self, node: Node) -> bool:
        try:
            if self.classes and not isinstance(node, self.classes):
                return False

            if self.attrs:
                if not isinstance(node, nodes.Element):
                    return False

                for key, value in self.attrs.items():
                    if key not in node:
                        return False
                    elif value is Any:
                        continue
                    elif node.get(key) != value:
                        return False

            return True
        except Exception:
            # for non-Element nodes
            return False

    def __call__(self, node: Node) -> bool:
        return self.match(node)

    def findall(self, node: Node) -> Iterator[N]:
        """An alternative to `Node.findall` with improved type safety.

        While the `NodeMatcher` object can be used as an argument to `Node.findall`, doing so
        confounds type checkers' ability to determine the return type of the iterator.
        """
        for found in node.findall(self):
            yield cast(N, found)


def get_full_module_name(node: Node) -> str:
    """
    Return full module dotted path like: 'docutils.nodes.paragraph'

    :param nodes.Node node: target node
    :return: full module dotted path
    """
    return f'{node.__module__}.{node.__class__.__name__}'


def repr_domxml(node: Node, length: int = 80) -> str:
    """
    return DOM XML representation of the specified node like:
    '<paragraph translatable="False"><inline classes="versionadded">Added in version...'

    :param nodes.Node node: target node
    :param int length:
       length of return value to be striped. if false-value is specified, repr_domxml
       returns full of DOM XML representation.
    :return: DOM XML representation
    """
    try:
        text = node.asdom().toxml()
    except Exception:
        text = str(node)
    if length and len(text) > length:
        text = text[:length] + '...'
    return text


def apply_source_workaround(node: Element) -> None:
    # workaround: nodes.term have wrong rawsource if classifier is specified.
    # The behavior of docutils-0.11, 0.12 is:
    # * when ``term text : classifier1 : classifier2`` is specified,
    # * rawsource of term node will have: ``term text : classifier1 : classifier2``
    # * rawsource of classifier node will be None
    if isinstance(node, nodes.classifier) and not node.rawsource:
        logger.debug('[i18n] PATCH: %r to have source, line and rawsource: %s',
                     get_full_module_name(node), repr_domxml(node))
        definition_list_item = node.parent
        node.source = definition_list_item.source
        node.line = definition_list_item.line - 1  # type: ignore[operator]
        node.rawsource = node.astext()  # set 'classifier1' (or 'classifier2')
    elif isinstance(node, nodes.classifier) and not node.source:
        # docutils-0.15 fills in rawsource attribute, but not in source.
        node.source = node.parent.source
    if isinstance(node, nodes.image) and node.source is None:
        logger.debug('[i18n] PATCH: %r to have source, line: %s',
                     get_full_module_name(node), repr_domxml(node))
        node.source, node.line = node.parent.source, node.parent.line
    if isinstance(node, nodes.title) and node.source is None:
        logger.debug('[i18n] PATCH: %r to have source: %s',
                     get_full_module_name(node), repr_domxml(node))
        node.source, node.line = node.parent.source, node.parent.line
    if isinstance(node, nodes.term):
        logger.debug('[i18n] PATCH: %r to have rawsource: %s',
                     get_full_module_name(node), repr_domxml(node))
        # strip classifier from rawsource of term
        for classifier in reversed(list(node.parent.findall(nodes.classifier))):
            node.rawsource = re.sub(r'\s*:\s*%s' % re.escape(classifier.astext()),
                                    '', node.rawsource)
    if isinstance(node, nodes.topic) and node.source is None:
        # docutils-0.18 does not fill the source attribute of topic
        logger.debug('[i18n] PATCH: %r to have source, line: %s',
                     get_full_module_name(node), repr_domxml(node))
        node.source, node.line = node.parent.source, node.parent.line

    # workaround: literal_block under bullet list (#4913)
    if isinstance(node, nodes.literal_block) and node.source is None:
        with contextlib.suppress(ValueError):
            node.source = get_node_source(node)

    # workaround: recommonmark-0.2.0 doesn't set rawsource attribute
    if not node.rawsource:
        node.rawsource = node.astext()

    if node.source and node.rawsource:
        return

    # workaround: some docutils nodes doesn't have source, line.
    if (isinstance(node, (
            nodes.rubric,  # #1305 rubric directive
            nodes.line,  # #1477 line node
            nodes.image,  # #3093 image directive in substitution
            nodes.field_name,  # #3335 field list syntax
    ))):
        logger.debug('[i18n] PATCH: %r to have source and line: %s',
                     get_full_module_name(node), repr_domxml(node))
        try:
            node.source = get_node_source(node)
        except ValueError:
            node.source = ''
        node.line = 0  # need fix docutils to get `node.line`
        return


IGNORED_NODES = (
    nodes.Invisible,
    nodes.literal_block,
    nodes.doctest_block,
    addnodes.versionmodified,
    # XXX there are probably more
)


def is_translatable(node: Node) -> bool:
    if isinstance(node, addnodes.translatable):
        return True

    # image node marked as translatable or having alt text
    if isinstance(node, nodes.image) and (node.get('translatable') or node.get('alt')):
        return True

    if isinstance(node, nodes.Inline) and 'translatable' not in node:  # type: ignore[operator]
        # inline node must not be translated if 'translatable' is not set
        return False

    if isinstance(node, nodes.TextElement):
        if not node.source:
            logger.debug('[i18n] SKIP %r because no node.source: %s',
                         get_full_module_name(node), repr_domxml(node))
            return False  # built-in message
        if isinstance(node, IGNORED_NODES) and 'translatable' not in node:
            logger.debug("[i18n] SKIP %r because node is in IGNORED_NODES "
                         "and no node['translatable']: %s",
                         get_full_module_name(node), repr_domxml(node))
            return False
        if not node.get('translatable', True):
            # not(node['translatable'] == True or node['translatable'] is None)
            logger.debug("[i18n] SKIP %r because not node['translatable']: %s",
                         get_full_module_name(node), repr_domxml(node))
            return False
        # <field_name>orphan</field_name>
        # XXX ignore all metadata (== docinfo)
        if isinstance(node, nodes.field_name) and (node.children[0] == 'orphan'):
            logger.debug('[i18n] SKIP %r because orphan node: %s',
                         get_full_module_name(node), repr_domxml(node))
            return False
        return True

    return isinstance(node, nodes.meta)


LITERAL_TYPE_NODES = (
    nodes.literal_block,
    nodes.doctest_block,
    nodes.math_block,
    nodes.raw,
)
IMAGE_TYPE_NODES = (
    nodes.image,
)


def extract_messages(doctree: Element) -> Iterable[tuple[Element, str]]:
    """Extract translatable messages from a document tree."""
    for node in doctree.findall(is_translatable):
        if isinstance(node, addnodes.translatable):
            for msg in node.extract_original_messages():
                yield node, msg  # type: ignore[misc]
            continue
        if isinstance(node, LITERAL_TYPE_NODES):
            msg = node.rawsource
            if not msg:
                msg = node.astext()
        elif isinstance(node, nodes.image):
            if node.get('alt'):
                yield node, node['alt']
            if node.get('translatable'):
                image_uri = node.get('original_uri', node['uri'])
                msg = f'.. image:: {image_uri}'
            else:
                msg = ''
        elif isinstance(node, nodes.meta):
            msg = node["content"]
        else:
            msg = node.rawsource.replace('\n', ' ').strip()  # type: ignore[attr-defined]

        # XXX nodes rendering empty are likely a bug in sphinx.addnodes
        if msg:
            yield node, msg  # type: ignore[misc]


def get_node_source(node: Element) -> str:
    for pnode in traverse_parent(node):
        if pnode.source:
            return pnode.source
    msg = 'node source not found'
    raise ValueError(msg)


def get_node_line(node: Element) -> int:
    for pnode in traverse_parent(node):
        if pnode.line:
            return pnode.line
    msg = 'node line not found'
    raise ValueError(msg)


def traverse_parent(node: Element, cls: Any = None) -> Iterable[Element]:
    while node:
        if cls is None or isinstance(node, cls):
            yield node
        node = node.parent


def get_prev_node(node: Node) -> Node | None:
    pos = node.parent.index(node)
    if pos > 0:
        return node.parent[pos - 1]
    else:
        return None


def traverse_translatable_index(
    doctree: Element,
) -> Iterable[tuple[Element, list[tuple[str, str, str, str, str | None]]]]:
    """Traverse translatable index node from a document tree."""
    matcher = NodeMatcher(addnodes.index, inline=False)
    for node in matcher.findall(doctree):
        if 'raw_entries' in node:
            entries = node['raw_entries']
        else:
            entries = node['entries']
        yield node, entries


def nested_parse_with_titles(state: Any, content: StringList, node: Node,
                             content_offset: int = 0) -> str:
    """Version of state.nested_parse() that allows titles and does not require
    titles to have the same decoration as the calling document.

    This is useful when the parsed content comes from a completely different
    context, such as docstrings.
    """
    # hack around title style bookkeeping
    surrounding_title_styles = state.memo.title_styles
    surrounding_section_level = state.memo.section_level
    state.memo.title_styles = []
    state.memo.section_level = 0
    try:
        return state.nested_parse(content, content_offset, node, match_titles=1)
    finally:
        state.memo.title_styles = surrounding_title_styles
        state.memo.section_level = surrounding_section_level


def clean_astext(node: Element) -> str:
    """Like node.astext(), but ignore images."""
    node = node.deepcopy()
    for img in node.findall(nodes.image):
        img['alt'] = ''
    for raw in list(node.findall(nodes.raw)):
        raw.parent.remove(raw)
    return node.astext()


def split_explicit_title(text: str) -> tuple[bool, str, str]:
    """Split role content into title and target, if given."""
    match = explicit_title_re.match(text)
    if match:
        return True, match.group(1), match.group(2)
    return False, text, text


indextypes = [
    'single', 'pair', 'double', 'triple', 'see', 'seealso',
]


def process_index_entry(entry: str, targetid: str,
                        ) -> list[tuple[str, str, str, str, str | None]]:
    from sphinx.domains.python import pairindextypes

    indexentries: list[tuple[str, str, str, str, str | None]] = []
    entry = entry.strip()
    oentry = entry
    main = ''
    if entry.startswith('!'):
        main = 'main'
        entry = entry[1:].lstrip()
    for index_type in pairindextypes:
        if entry.startswith(f'{index_type}:'):
            value = entry[len(index_type) + 1:].strip()
            value = f'{pairindextypes[index_type]}; {value}'
            # xref RemovedInSphinx90Warning
            logger.warning(__('%r is deprecated for index entries (from entry %r). '
                              "Use 'pair: %s' instead."),
                           index_type, entry, value, type='index')
            indexentries.append(('pair', value, targetid, main, None))
            break
    else:
        for index_type in indextypes:
            if entry.startswith(f'{index_type}:'):
                value = entry[len(index_type) + 1:].strip()
                if index_type == 'double':
                    index_type = 'pair'
                indexentries.append((index_type, value, targetid, main, None))
                break
        # shorthand notation for single entries
        else:
            for value in oentry.split(','):
                value = value.strip()
                main = ''
                if value.startswith('!'):
                    main = 'main'
                    value = value[1:].lstrip()
                if not value:
                    continue
                indexentries.append(('single', value, targetid, main, None))
    return indexentries


def inline_all_toctrees(
    builder: Builder,
    docnameset: set[str],
    docname: str,
    tree: nodes.document,
    colorfunc: Callable[[str], str],
    traversed: list[str],
) -> nodes.document:
    """Inline all toctrees in the *tree*.

    Record all docnames in *docnameset*, and output docnames with *colorfunc*.
    """
    tree = tree.deepcopy()
    for toctreenode in list(tree.findall(addnodes.toctree)):
        newnodes = []
        includefiles = map(str, toctreenode['includefiles'])
        for includefile in includefiles:
            if includefile not in traversed:
                try:
                    traversed.append(includefile)
                    logger.info(colorfunc(includefile) + " ", nonl=True)
                    subtree = inline_all_toctrees(builder, docnameset, includefile,
                                                  builder.env.get_doctree(includefile),
                                                  colorfunc, traversed)
                    docnameset.add(includefile)
                except Exception:
                    logger.warning(__('toctree contains ref to nonexisting file %r'),
                                   includefile, location=docname)
                else:
                    sof = addnodes.start_of_file(docname=includefile)
                    sof.children = subtree.children
                    for sectionnode in sof.findall(nodes.section):
                        if 'docname' not in sectionnode:
                            sectionnode['docname'] = includefile
                    newnodes.append(sof)
        toctreenode.parent.replace(toctreenode, newnodes)
    return tree


def _make_id(string: str) -> str:
    """Convert `string` into an identifier and return it.

    This function is a modified version of ``docutils.nodes.make_id()`` of
    docutils-0.16.

    Changes:

    * Allow to use capital alphabet characters
    * Allow to use dots (".") and underscores ("_") for an identifier
      without a leading character.

    # Author: David Goodger <goodger@python.org>
    # Maintainer: docutils-develop@lists.sourceforge.net
    # Copyright: This module has been placed in the public domain.
    """
    id = string.translate(_non_id_translate_digraphs)
    id = id.translate(_non_id_translate)
    # get rid of non-ascii characters.
    # 'ascii' lowercase to prevent problems with turkish locale.
    id = unicodedata.normalize('NFKD', id).encode('ascii', 'ignore').decode('ascii')
    # shrink runs of whitespace and replace by hyphen
    id = _non_id_chars.sub('-', ' '.join(id.split()))
    id = _non_id_at_ends.sub('', id)
    return str(id)


_non_id_chars = re.compile('[^a-zA-Z0-9._]+')
_non_id_at_ends = re.compile('^[-0-9._]+|-+$')
_non_id_translate = {
    0x00f8: 'o',       # o with stroke
    0x0111: 'd',       # d with stroke
    0x0127: 'h',       # h with stroke
    0x0131: 'i',       # dotless i
    0x0142: 'l',       # l with stroke
    0x0167: 't',       # t with stroke
    0x0180: 'b',       # b with stroke
    0x0183: 'b',       # b with topbar
    0x0188: 'c',       # c with hook
    0x018c: 'd',       # d with topbar
    0x0192: 'f',       # f with hook
    0x0199: 'k',       # k with hook
    0x019a: 'l',       # l with bar
    0x019e: 'n',       # n with long right leg
    0x01a5: 'p',       # p with hook
    0x01ab: 't',       # t with palatal hook
    0x01ad: 't',       # t with hook
    0x01b4: 'y',       # y with hook
    0x01b6: 'z',       # z with stroke
    0x01e5: 'g',       # g with stroke
    0x0225: 'z',       # z with hook
    0x0234: 'l',       # l with curl
    0x0235: 'n',       # n with curl
    0x0236: 't',       # t with curl
    0x0237: 'j',       # dotless j
    0x023c: 'c',       # c with stroke
    0x023f: 's',       # s with swash tail
    0x0240: 'z',       # z with swash tail
    0x0247: 'e',       # e with stroke
    0x0249: 'j',       # j with stroke
    0x024b: 'q',       # q with hook tail
    0x024d: 'r',       # r with stroke
    0x024f: 'y',       # y with stroke
}
_non_id_translate_digraphs = {
    0x00df: 'sz',      # ligature sz
    0x00e6: 'ae',      # ae
    0x0153: 'oe',      # ligature oe
    0x0238: 'db',      # db digraph
    0x0239: 'qp',      # qp digraph
}


def make_id(env: BuildEnvironment, document: nodes.document,
            prefix: str = '', term: str | None = None) -> str:
    """Generate an appropriate node_id for given *prefix* and *term*."""
    node_id = None
    if prefix:
        idformat = prefix + "-%s"
    else:
        idformat = (document.settings.id_prefix or "id") + "%s"

    # try to generate node_id by *term*
    if prefix and term:
        node_id = _make_id(idformat % term)
        if node_id == prefix:
            # *term* is not good to generate a node_id.
            node_id = None
    elif term:
        node_id = _make_id(term)
        if node_id == '':
            node_id = None  # fallback to None

    while node_id is None or node_id in document.ids:
        node_id = idformat % env.new_serialno(prefix)

    return node_id


def find_pending_xref_condition(node: addnodes.pending_xref, condition: str,
                                ) -> Element | None:
    """Pick matched pending_xref_condition node up from the pending_xref."""
    for subnode in node:
        if (isinstance(subnode, addnodes.pending_xref_condition) and
                subnode.get('condition') == condition):
            return subnode
    return None


def make_refnode(builder: Builder, fromdocname: str, todocname: str, targetid: str | None,
                 child: Node | list[Node], title: str | None = None,
                 ) -> nodes.reference:
    """Shortcut to create a reference node."""
    node = nodes.reference('', '', internal=True)
    if fromdocname == todocname and targetid:
        node['refid'] = targetid
    else:
        if targetid:
            node['refuri'] = (builder.get_relative_uri(fromdocname, todocname) +
                              '#' + targetid)
        else:
            node['refuri'] = builder.get_relative_uri(fromdocname, todocname)
    if title:
        node['reftitle'] = title
    node += child
    return node


def set_source_info(directive: Directive, node: Node) -> None:
    node.source, node.line = \
        directive.state_machine.get_source_and_line(directive.lineno)


def set_role_source_info(inliner: Inliner, lineno: int, node: Node) -> None:
    gsal = inliner.reporter.get_source_and_line  # type: ignore[attr-defined]
    node.source, node.line = gsal(lineno)


def copy_source_info(src: Element, dst: Element) -> None:
    with contextlib.suppress(ValueError):
        dst.source = get_node_source(src)
        dst.line = get_node_line(src)


NON_SMARTQUOTABLE_PARENT_NODES = (
    nodes.FixedTextElement,
    nodes.literal,
    nodes.math,
    nodes.image,
    nodes.raw,
    nodes.problematic,
    addnodes.not_smartquotable,
)


def is_smartquotable(node: Node) -> bool:
    """Check whether the node is smart-quotable or not."""
    for pnode in traverse_parent(node.parent):
        if isinstance(pnode, NON_SMARTQUOTABLE_PARENT_NODES):
            return False
        if pnode.get('support_smartquotes', None) is False:
            return False

    return getattr(node, 'support_smartquotes', None) is not False


def process_only_nodes(document: Node, tags: Tags) -> None:
    """Filter ``only`` nodes which do not match *tags*."""
    for node in document.findall(addnodes.only):
        if _only_node_keep_children(node, tags):
            node.replace_self(node.children or nodes.comment())
        else:
            # A comment on the comment() nodes being inserted: replacing by [] would
            # result in a "Losing ids" exception if there is a target node before
            # the only node, so we make sure docutils can transfer the id to
            # something, even if it's just a comment and will lose the id anyway...
            node.replace_self(nodes.comment())


def _only_node_keep_children(node: addnodes.only, tags: Tags) -> bool:
    """Keep children if tags match or error."""
    try:
        return tags.eval_condition(node['expr'])
    except Exception as err:
        logger.warning(
            __('exception while evaluating only directive expression: %s'),
            err,
            location=node)
        return True


def _copy_except__document(el: Element) -> Element:
    """Monkey-patch ```nodes.Element.copy``` to not copy the ``_document``
    attribute.

    xref: https://github.com/sphinx-doc/sphinx/issues/11116#issuecomment-1376767086
    """
    newnode = object.__new__(el.__class__)
    # set in Element.__init__()
    newnode.children = []
    newnode.rawsource = el.rawsource
    newnode.tagname = el.tagname
    # copied in Element.copy()
    newnode.attributes = {k: (v
                              if k not in {'ids', 'classes', 'names', 'dupnames', 'backrefs'}
                              else v[:])
                          for k, v in el.attributes.items()}
    newnode.line = el.line
    newnode.source = el.source
    return newnode


nodes.Element.copy = _copy_except__document  # type: ignore[assignment]


def _deepcopy(el: Element) -> Element:
    """Monkey-patch ```nodes.Element.deepcopy``` for speed."""
    newnode = el.copy()
    newnode.children = [child.deepcopy() for child in el.children]
    for child in newnode.children:
        child.parent = newnode
        if el.document:
            child.document = el.document
            if child.source is None:
                child.source = el.document.current_source
            if child.line is None:
                child.line = el.document.current_line
    return newnode


nodes.Element.deepcopy = _deepcopy  # type: ignore[assignment]