From aa57244e71bf7e94f14eb66649ec118f830e9616 Mon Sep 17 00:00:00 2001 From: Taneli Hukkinen Date: Fri, 26 Feb 2021 03:28:50 +0200 Subject: [PATCH 1/6] Attempt to make SyntaxTreeNode type annotations work when subclassing --- markdown_it/tree.py | 48 ++++++++++++++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/markdown_it/tree.py b/markdown_it/tree.py index c0036139..5fe7aa06 100644 --- a/markdown_it/tree.py +++ b/markdown_it/tree.py @@ -2,13 +2,27 @@ This module is not part of upstream JavaScript markdown-it. """ -from typing import NamedTuple, Sequence, Tuple, Dict, List, Optional, Any +from typing import ( + NamedTuple, + Sequence, + Tuple, + Dict, + List, + Optional, + Any, + TypeVar, + Type, + Generic, +) from .token import Token from .utils import _removesuffix -class SyntaxTreeNode: +_T = TypeVar("_T", bound="SyntaxTreeNodeBase") + + +class SyntaxTreeNodeBase(Generic[_T]): """A Markdown syntax tree node. A class that can be used to construct a tree representation of a linear @@ -35,17 +49,17 @@ def __init__(self) -> None: self.token: Optional[Token] = None # Only containers have nester tokens - self.nester_tokens: Optional[SyntaxTreeNode._NesterTokens] = None + self.nester_tokens: Optional[SyntaxTreeNodeBase._NesterTokens] = None # Root node does not have self.parent - self.parent: Optional["SyntaxTreeNode"] = None + self.parent: Optional[_T] = None # Empty list unless a non-empty container, or unnested token that has # children (i.e. inline or img) - self.children: List["SyntaxTreeNode"] = [] + self.children: List[_T] = [] @classmethod - def from_tokens(cls, tokens: Sequence[Token]) -> "SyntaxTreeNode": + def from_tokens(cls: Type[_T], tokens: Sequence[Token]) -> _T: """Instantiate a `SyntaxTreeNode` from a token stream. This is the standard method for instantiating `SyntaxTreeNode`. @@ -54,12 +68,10 @@ def from_tokens(cls, tokens: Sequence[Token]) -> "SyntaxTreeNode": root._set_children_from_tokens(tokens) return root - def to_tokens(self) -> List[Token]: + def to_tokens(self: _T) -> List[Token]: """Recover the linear token stream.""" - def recursive_collect_tokens( - node: "SyntaxTreeNode", token_list: List[Token] - ) -> None: + def recursive_collect_tokens(node: _T, token_list: List[Token]) -> None: if node.type == "root": for child in node.children: recursive_collect_tokens(child, token_list) @@ -87,7 +99,7 @@ def is_nested(self) -> bool: return bool(self.nester_tokens) @property - def siblings(self) -> Sequence["SyntaxTreeNode"]: + def siblings(self: _T) -> Sequence[_T]: """Get siblings of the node. Gets the whole group of siblings, including self. @@ -113,7 +125,7 @@ def type(self) -> str: return _removesuffix(self.nester_tokens.opening.type, "_open") @property - def next_sibling(self) -> Optional["SyntaxTreeNode"]: + def next_sibling(self: _T) -> Optional[_T]: """Get the next node in the sequence of siblings. Returns `None` if this is the last sibling. @@ -124,7 +136,7 @@ def next_sibling(self) -> Optional["SyntaxTreeNode"]: return None @property - def previous_sibling(self) -> Optional["SyntaxTreeNode"]: + def previous_sibling(self: _T) -> Optional[_T]: """Get the previous node in the sequence of siblings. Returns `None` if this is the first sibling. @@ -135,11 +147,11 @@ def previous_sibling(self) -> Optional["SyntaxTreeNode"]: return None def _make_child( - self, + self: _T, *, token: Optional[Token] = None, nester_tokens: Optional[_NesterTokens] = None, - ) -> "SyntaxTreeNode": + ) -> _T: """Make and return a child node for `self`.""" if token and nester_tokens or not token and not nester_tokens: raise ValueError("must specify either `token` or `nester_tokens`") @@ -177,7 +189,7 @@ def _set_children_from_tokens(self, tokens: Sequence[Token]) -> None: raise ValueError(f"unclosed tokens starting {nested_tokens[0]}") child = self._make_child( - nester_tokens=SyntaxTreeNode._NesterTokens( + nester_tokens=SyntaxTreeNodeBase._NesterTokens( nested_tokens[0], nested_tokens[-1] ) ) @@ -260,3 +272,7 @@ def hidden(self) -> bool: """If it's true, ignore this element when rendering. Used for tight lists to hide paragraphs.""" return self._attribute_token().hidden + + +class SyntaxTreeNode(SyntaxTreeNodeBase["SyntaxTreeNode"]): + pass From fce3d214efc686b03907eea1d543a59edb944e03 Mon Sep 17 00:00:00 2001 From: Taneli Hukkinen Date: Fri, 26 Feb 2021 05:34:33 +0200 Subject: [PATCH 2/6] Make _parent and _children properties --- markdown_it/tree.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/markdown_it/tree.py b/markdown_it/tree.py index 5fe7aa06..9d4259c0 100644 --- a/markdown_it/tree.py +++ b/markdown_it/tree.py @@ -12,17 +12,16 @@ Any, TypeVar, Type, - Generic, ) from .token import Token from .utils import _removesuffix -_T = TypeVar("_T", bound="SyntaxTreeNodeBase") +_T = TypeVar("_T", bound="SyntaxTreeNode") -class SyntaxTreeNodeBase(Generic[_T]): +class SyntaxTreeNode: """A Markdown syntax tree node. A class that can be used to construct a tree representation of a linear @@ -49,14 +48,14 @@ def __init__(self) -> None: self.token: Optional[Token] = None # Only containers have nester tokens - self.nester_tokens: Optional[SyntaxTreeNodeBase._NesterTokens] = None + self.nester_tokens: Optional[SyntaxTreeNode._NesterTokens] = None # Root node does not have self.parent - self.parent: Optional[_T] = None + self._parent: Any = None # Optional[_T] # Empty list unless a non-empty container, or unnested token that has # children (i.e. inline or img) - self.children: List[_T] = [] + self._children: list = [] # List[_T] @classmethod def from_tokens(cls: Type[_T], tokens: Sequence[Token]) -> _T: @@ -88,6 +87,14 @@ def recursive_collect_tokens(node: _T, token_list: List[Token]) -> None: recursive_collect_tokens(self, tokens) return tokens + @property + def children(self: _T) -> Sequence[_T]: + return self._children + + @property + def parent(self: _T) -> Optional[_T]: + return self._parent + @property def is_nested(self) -> bool: """Is this node nested?. @@ -160,8 +167,8 @@ def _make_child( child.token = token else: child.nester_tokens = nester_tokens - child.parent = self - self.children.append(child) + child._parent = self + self._children.append(child) return child def _set_children_from_tokens(self, tokens: Sequence[Token]) -> None: @@ -189,7 +196,7 @@ def _set_children_from_tokens(self, tokens: Sequence[Token]) -> None: raise ValueError(f"unclosed tokens starting {nested_tokens[0]}") child = self._make_child( - nester_tokens=SyntaxTreeNodeBase._NesterTokens( + nester_tokens=SyntaxTreeNode._NesterTokens( nested_tokens[0], nested_tokens[-1] ) ) @@ -272,7 +279,3 @@ def hidden(self) -> bool: """If it's true, ignore this element when rendering. Used for tight lists to hide paragraphs.""" return self._attribute_token().hidden - - -class SyntaxTreeNode(SyntaxTreeNodeBase["SyntaxTreeNode"]): - pass From aea99cedb7157f30f30e177be46f2524319fd4ee Mon Sep 17 00:00:00 2001 From: Taneli Hukkinen Date: Sat, 27 Feb 2021 03:30:45 +0200 Subject: [PATCH 3/6] Move _NesterTokens to module level now that SyntaxTreeNode has its own module --- markdown_it/tree.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/markdown_it/tree.py b/markdown_it/tree.py index 9d4259c0..dbe7b38a 100644 --- a/markdown_it/tree.py +++ b/markdown_it/tree.py @@ -18,6 +18,11 @@ from .utils import _removesuffix +class _NesterTokens(NamedTuple): + opening: Token + closing: Token + + _T = TypeVar("_T", bound="SyntaxTreeNode") @@ -35,10 +40,6 @@ class SyntaxTreeNode: between """ - class _NesterTokens(NamedTuple): - opening: Token - closing: Token - def __init__(self) -> None: """Initialize a root node with no children. @@ -48,7 +49,7 @@ def __init__(self) -> None: self.token: Optional[Token] = None # Only containers have nester tokens - self.nester_tokens: Optional[SyntaxTreeNode._NesterTokens] = None + self.nester_tokens: Optional[_NesterTokens] = None # Root node does not have self.parent self._parent: Any = None # Optional[_T] @@ -196,9 +197,7 @@ def _set_children_from_tokens(self, tokens: Sequence[Token]) -> None: raise ValueError(f"unclosed tokens starting {nested_tokens[0]}") child = self._make_child( - nester_tokens=SyntaxTreeNode._NesterTokens( - nested_tokens[0], nested_tokens[-1] - ) + nester_tokens=_NesterTokens(nested_tokens[0], nested_tokens[-1]) ) child._set_children_from_tokens(nested_tokens[1:-1]) From 1b4e817faa8549d6ff265f1c2d857a2b4ca11aca Mon Sep 17 00:00:00 2001 From: Taneli Hukkinen Date: Sun, 28 Feb 2021 04:40:37 +0200 Subject: [PATCH 4/6] Add setters for `parent` and `children` --- markdown_it/tree.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/markdown_it/tree.py b/markdown_it/tree.py index dbe7b38a..5acdff2a 100644 --- a/markdown_it/tree.py +++ b/markdown_it/tree.py @@ -52,11 +52,11 @@ def __init__(self) -> None: self.nester_tokens: Optional[_NesterTokens] = None # Root node does not have self.parent - self._parent: Any = None # Optional[_T] + self.parent = None # Empty list unless a non-empty container, or unnested token that has # children (i.e. inline or img) - self._children: list = [] # List[_T] + self.children = [] @classmethod def from_tokens(cls: Type[_T], tokens: Sequence[Token]) -> _T: @@ -89,13 +89,21 @@ def recursive_collect_tokens(node: _T, token_list: List[Token]) -> None: return tokens @property - def children(self: _T) -> Sequence[_T]: + def children(self: _T) -> List[_T]: return self._children + @children.setter + def children(self: _T, value: List[_T]) -> None: + self._children = value + @property def parent(self: _T) -> Optional[_T]: return self._parent + @parent.setter + def parent(self: _T, value: Optional[_T]) -> None: + self._parent = value + @property def is_nested(self) -> bool: """Is this node nested?. @@ -168,8 +176,8 @@ def _make_child( child.token = token else: child.nester_tokens = nester_tokens - child._parent = self - self._children.append(child) + child.parent = self + self.children.append(child) return child def _set_children_from_tokens(self, tokens: Sequence[Token]) -> None: From e5f709bbc153d791cf6dbdb641ff6ce3c154dd77 Mon Sep 17 00:00:00 2001 From: Taneli Hukkinen Date: Thu, 4 Mar 2021 02:28:45 +0200 Subject: [PATCH 5/6] Address PR review --- markdown_it/tree.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/markdown_it/tree.py b/markdown_it/tree.py index 4c984860..007ed833 100644 --- a/markdown_it/tree.py +++ b/markdown_it/tree.py @@ -24,7 +24,7 @@ class _NesterTokens(NamedTuple): closing: Token -_T = TypeVar("_T", bound="SyntaxTreeNode") +_NodeType = TypeVar("_NodeType", bound="SyntaxTreeNode") class SyntaxTreeNode: @@ -53,11 +53,11 @@ def __init__(self) -> None: self.nester_tokens: Optional[_NesterTokens] = None # Root node does not have self.parent - self.parent = None + self._parent: Any = None # Empty list unless a non-empty container, or unnested token that has # children (i.e. inline or img) - self.children = [] + self._children: list = [] def __repr__(self) -> str: return f"{type(self).__name__}({self.type})" @@ -66,7 +66,7 @@ def __getitem__(self, item: int) -> "SyntaxTreeNode": return self.children[item] @classmethod - def from_tokens(cls: Type[_T], tokens: Sequence[Token]) -> _T: + def from_tokens(cls: Type[_NodeType], tokens: Sequence[Token]) -> _NodeType: """Instantiate a `SyntaxTreeNode` from a token stream. This is the standard method for instantiating `SyntaxTreeNode`. @@ -75,10 +75,10 @@ def from_tokens(cls: Type[_T], tokens: Sequence[Token]) -> _T: root._set_children_from_tokens(tokens) return root - def to_tokens(self: _T) -> List[Token]: + def to_tokens(self: _NodeType) -> List[Token]: """Recover the linear token stream.""" - def recursive_collect_tokens(node: _T, token_list: List[Token]) -> None: + def recursive_collect_tokens(node: _NodeType, token_list: List[Token]) -> None: if node.type == "root": for child in node.children: recursive_collect_tokens(child, token_list) @@ -96,19 +96,19 @@ def recursive_collect_tokens(node: _T, token_list: List[Token]) -> None: return tokens @property - def children(self: _T) -> List[_T]: + def children(self: _NodeType) -> List[_NodeType]: return self._children @children.setter - def children(self: _T, value: List[_T]) -> None: + def children(self: _NodeType, value: List[_NodeType]) -> None: self._children = value @property - def parent(self: _T) -> Optional[_T]: + def parent(self: _NodeType) -> Optional[_NodeType]: return self._parent @parent.setter - def parent(self: _T, value: Optional[_T]) -> None: + def parent(self: _NodeType, value: Optional[_NodeType]) -> None: self._parent = value @property @@ -127,7 +127,7 @@ def is_nested(self) -> bool: return bool(self.nester_tokens) @property - def siblings(self: _T) -> Sequence[_T]: + def siblings(self: _NodeType) -> Sequence[_NodeType]: """Get siblings of the node. Gets the whole group of siblings, including self. @@ -153,7 +153,7 @@ def type(self) -> str: return _removesuffix(self.nester_tokens.opening.type, "_open") @property - def next_sibling(self: _T) -> Optional[_T]: + def next_sibling(self: _NodeType) -> Optional[_NodeType]: """Get the next node in the sequence of siblings. Returns `None` if this is the last sibling. @@ -164,7 +164,7 @@ def next_sibling(self: _T) -> Optional[_T]: return None @property - def previous_sibling(self: _T) -> Optional[_T]: + def previous_sibling(self: _NodeType) -> Optional[_NodeType]: """Get the previous node in the sequence of siblings. Returns `None` if this is the first sibling. @@ -175,11 +175,11 @@ def previous_sibling(self: _T) -> Optional[_T]: return None def _make_child( - self: _T, + self: _NodeType, *, token: Optional[Token] = None, nester_tokens: Optional[_NesterTokens] = None, - ) -> _T: + ) -> _NodeType: """Make and return a child node for `self`.""" if token and nester_tokens or not token and not nester_tokens: raise ValueError("must specify either `token` or `nester_tokens`") From c0b698af9318ccc8c8c4f3b91a1f5a0a2d4739a4 Mon Sep 17 00:00:00 2001 From: Taneli Hukkinen Date: Thu, 4 Mar 2021 16:36:07 +0200 Subject: [PATCH 6/6] Fix __getitem__ type annotation --- markdown_it/tree.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/markdown_it/tree.py b/markdown_it/tree.py index 007ed833..58398d9a 100644 --- a/markdown_it/tree.py +++ b/markdown_it/tree.py @@ -13,6 +13,8 @@ Any, TypeVar, Type, + overload, + Union, ) from .token import Token @@ -62,7 +64,17 @@ def __init__(self) -> None: def __repr__(self) -> str: return f"{type(self).__name__}({self.type})" - def __getitem__(self, item: int) -> "SyntaxTreeNode": + @overload + def __getitem__(self: _NodeType, item: int) -> _NodeType: + ... + + @overload + def __getitem__(self: _NodeType, item: slice) -> List[_NodeType]: + ... + + def __getitem__( + self: _NodeType, item: Union[int, slice] + ) -> Union[_NodeType, List[_NodeType]]: return self.children[item] @classmethod