util: testing: consoletest: parser: Add basic .rst parser

Signed-off-by: John Andersen <johnandersenpdx@gmail.com>

Author: pdxjohnny

dffml/util/testing/consoletest/parser.py

@@ -0,0 +1,154 @@
+import dataclasses
+from typing import Dict, List, Union, Any
+
+
+@dataclasses.dataclass
+class Node:
+    directive: str
+    content: str
+    options: Dict[str, Union[bool, str]]
+    node: Dict[str, Any]
+
+
+class ParseSingleNodeFailure(Exception):
+    """
+    Raised when a node could not be parsed
+    """
+
+
+class ParseNodesFailure(Exception):
+    """
+    Raised when text could not be parsed
+    """
+
+
+def get_indent(line) -> str:
+    return line[: len(line) - len(line.lstrip())]
+
+
+def remove_indent(lines: str) -> List[str]:
+    # Find the lowest indentation level and remove it from all lines
+    indents = list(
+        map(
+            lambda line: len(get_indent(line)),
+            filter(lambda line: bool(line.strip()), lines),
+        )
+    )
+    if not indents:
+        return lines
+
+    lowest_ident = min(indents)
+    for i, line in enumerate(lines):
+        lines[i] = line[lowest_ident:]
+
+    return lines
+
+
+def _parse_nodes(text: str) -> List[Node]:
+    """
+    Quick and dirty implementation of a .rst parser to extract directives
+    """
+    nodes = []
+
+    c_indent = ""
+    in_section = []
+    directive = ""
+    last_directive = ""
+    args = ""
+    last_args = ""
+    text_split = text.split("\n")
+    for i, line in enumerate(text_split):
+        indent = get_indent(line)
+
+        if line.strip().startswith(".. ") and "::" in line:
+            last_directive = directive
+            directive = line[line.index(".. ") + 3 : line.index("::")]
+            if not last_directive:
+                last_directive = directive
+            last_args = args
+            args = line[line.index("::") + 2 :]
+            if not last_args:
+                last_args = args
+            if in_section:
+                if not in_section[-1]:
+                    in_section.pop()
+                nodes.append((last_directive, last_args, in_section,))
+            in_section = [""]
+            c_indent = indent
+        elif in_section:
+            if len(indent) <= len(c_indent) and line.strip():
+                if not in_section[-1]:
+                    in_section.pop()
+                nodes.append((directive, args, in_section,))
+                if line.strip().startswith(".. ") and "::" in line:
+                    last_directive = directive
+                    directive = line[line.index(".. ") + 3 : line.index("::")]
+                    if not last_directive:
+                        last_directive = directive
+                    last_args = args
+                    args = line[line.index("::") + 2 :]
+                    if not last_args:
+                        last_args = args
+                    in_section = [""]
+                    c_indent = indent
+                else:
+                    in_section = []
+                    c_indent = ""
+            else:
+                in_section.append(line)
+
+    if in_section:
+        if not in_section[-1]:
+            in_section.pop()
+        nodes.append((directive, args, in_section,))
+
+    # Remove first blank line
+    nodes = [
+        (
+            directive,
+            list(map(lambda i: i.strip(), filter(bool, args.split()))),
+            in_section[1:],
+        )
+        for directive, args, in_section in nodes
+    ]
+
+    new_nodes = []
+
+    for directive, args, old_node in nodes:
+        new_node = Node(directive=directive, options={}, content="", node={})
+        option_lines = []
+
+        try:
+            if directive == "code-block":
+                new_node.content = old_node[old_node.index("") + 1 :]
+                option_lines = old_node[: old_node.index("")]
+            elif directive == "literalinclude":
+                option_lines = old_node
+                new_node.node["source"] = args[0]
+
+            # Parse the options
+            if option_lines:
+                option_lines = remove_indent(option_lines)
+                for option in option_lines:
+                    option_split = option.split(" ", maxsplit=1)
+                    if len(option_split) == 1:
+                        option_split.append(True)
+                    new_node.options[option_split[0][1:-1]] = option_split[1]
+
+            new_node.content = remove_indent(new_node.content)
+
+        except Exception as error:
+            raise ParseSingleNodeFailure(
+                f"Failed to parse directive({directive}), args({args}), old_node: {old_node}"
+            ) from error
+
+        new_nodes.append(new_node)
+
+    return new_nodes
+
+
+def parse_nodes(text: str) -> List[Node]:
+    try:
+        return _parse_nodes(text)
+    except Exception as error:
+        raise ParseNodesFailure(f"Failed to parse: {text}") from error

tests/docs/test_consoletest.py

@@ -1,5 +1,6 @@
 import os
 import sys
+import inspect
 import pathlib
 import tempfile
 import unittest
@@ -10,6 +11,7 @@
 from dffml.util.asynctestcase import AsyncTestCase
 
 from dffml.util.testing.consoletest.commands import *
+from dffml.util.testing.consoletest.parser import parse_nodes, Node
 
 
 # Root of DFFML source tree
@@ -170,6 +172,145 @@ def test_find_name(self):
         )
 
 
+class TestParser(unittest.TestCase):
+    def test_parse_nodes(self):
+        self.maxDiff = None
+        self.assertListEqual(
+            list(
+                filter(
+                    lambda node: node.directive
+                    in {"code-block", "literalinclude"},
+                    parse_nodes(
+                        inspect.cleandoc(
+                            r"""
+                .. code-block:: console
+                    :test:
+
+                    $ echo -e 'Hello\n\n\nWorld'
+                    Hello
+
+
+                    World
+
+                .. literalinclude:: some/file.py
+                    :filepath: myfile.py
+                    :test:
+
+                .. note::
+
+                    .. note::
+
+                        .. code-block:: console
+                            :test:
+                            :daemon: 8080
+
+                            $ echo -e 'Hello\n\n\n    World\n\n\nTest'
+                            Hello
+
+
+                                World
+
+
+                            Test
+
+                    .. code-block:: console
+
+                        $ echo -e 'Hello\n\n\n    World\n\n\n\n'
+                        Hello
+
+
+                            World
+
+
+
+                        $ echo 'feedface'
+                        feedface
+
+                    .. note::
+
+                        .. code-block:: console
+                            :test:
+
+                            $ echo feedface
+                            feedface
+
+                .. code-block:: console
+                    :test:
+
+                    $ echo feedface
+                    feedface
+                """
+                        )
+                    ),
+                )
+            ),
+            [
+                Node(
+                    directive="code-block",
+                    content=[
+                        r"$ echo -e 'Hello\n\n\nWorld'",
+                        "Hello",
+                        "",
+                        "",
+                        "World",
+                    ],
+                    options={"test": True},
+                    node={},
+                ),
+                Node(
+                    directive="literalinclude",
+                    content="",
+                    options={"filepath": "myfile.py", "test": True},
+                    node={"source": "some/file.py"},
+                ),
+                Node(
+                    directive="code-block",
+                    content=[
+                        r"$ echo -e 'Hello\n\n\n    World\n\n\nTest'",
+                        "Hello",
+                        "",
+                        "",
+                        "    World",
+                        "",
+                        "",
+                        "Test",
+                    ],
+                    options={"test": True, "daemon": "8080"},
+                    node={},
+                ),
+                Node(
+                    directive="code-block",
+                    content=[
+                        r"$ echo -e 'Hello\n\n\n    World\n\n\n\n'",
+                        "Hello",
+                        "",
+                        "",
+                        "    World",
+                        "",
+                        "",
+                        "",
+                        "$ echo 'feedface'",
+                        "feedface",
+                    ],
+                    options={},
+                    node={},
+                ),
+                Node(
+                    directive="code-block",
+                    content=["$ echo feedface", "feedface",],
+                    options={"test": True},
+                    node={},
+                ),
+                Node(
+                    directive="code-block",
+                    content=["$ echo feedface", "feedface",],
+                    options={"test": True},
+                    node={},
+                ),
+            ],
+        )
+
+
 ROOT_PATH = pathlib.Path(__file__).parent.parent.parent
 DOCS_PATH = ROOT_PATH / "docs"