From ef7c18a172a22dae93696e913ecadf9df5fdd655 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Wed, 19 Feb 2020 11:06:34 +1100 Subject: [PATCH] Add colon syntax for directive options. (#50) Also add error reporting for YAML block parsing. --- docs/using/syntax.md | 13 +++++ myst_parser/docutils_renderer.py | 79 +++++++++++++++++++++++++--- myst_parser/sphinx_parser.py | 4 +- pytest.ini | 4 ++ tests/sphinx_directives.json | 84 ++++++++++++++++++------------ tests/test_docutils_renderer.py | 88 ++++++++++++++++++++++++++------ 6 files changed, 214 insertions(+), 58 deletions(-) create mode 100644 pytest.ini diff --git a/docs/using/syntax.md b/docs/using/syntax.md index 7aa3fff0..b6129526 100644 --- a/docs/using/syntax.md +++ b/docs/using/syntax.md @@ -180,6 +180,19 @@ print('my 1st line') print(f'my {a}nd line') ``` +As a short-hand alternative, more closely resembling the reStructuredText syntax, options may also be denoted by an initial block, whereby all lines start with '`:`', for example: + +```` +```{code-block} python +:lineno-start: 10 +:emphasize-lines: 1, 3 + +a = 2 +print('my 1st line') +print(f'my {a}nd line') +``` +```` + ### Nesting directives You can nest directives by ensuring that the ticklines corresponding to the diff --git a/myst_parser/docutils_renderer.py b/myst_parser/docutils_renderer.py index 2fa3e673..e6a5019d 100644 --- a/myst_parser/docutils_renderer.py +++ b/myst_parser/docutils_renderer.py @@ -421,25 +421,88 @@ def render_role(self, token): self.current_node += problematic def render_directive(self, token): + """parse fenced code blocks as directives. + + Such a fenced code block starts with `{directive_name}`, + followed by arguments on the same line. + + Directive options are read from a YAML block, + if the first content line starts with `---`, e.g. + + :: + + ```{directive_name} arguments + --- + option1: name + option2: | + Longer text block + --- + content... + ``` + + Or the option block will be parsed if the first content line starts with `:`, + as a YAML block consisting of every line that starts with a `:`, e.g. + + :: + + ```{directive_name} arguments + :option1: name + :option2: other + + content... + ``` + + If the first line of a directive's content is blank, this will be stripped + from the content. + This is to allow for separation between the option block and content. + + """ name = token.language[1:-1] content = token.children[0].content options = {} + # get YAML options if content.startswith("---"): content = "\n".join(content.splitlines()[1:]) - # get YAML options match = re.search(r"^-{3,}", content, re.MULTILINE) if match: yaml_block = content[: match.start()] - content = content[match.end() :] # TODO advance line number + content = content[match.end() + 1 :] # TODO advance line number else: yaml_block = content content = "" try: options = yaml.safe_load(yaml_block) or {} - except yaml.parser.ParserError: - # TODO handle/report yaml parse error - pass - # TODO check options are an un-nested dict? + except (yaml.parser.ParserError, yaml.scanner.ScannerError) as error: + msg_node = self.document.reporter.system_message( + 3, "Directive options:\n" + str(error), line=token.range[0] + ) # 3 is ERROR level + msg_node += nodes.literal_block(yaml_block, yaml_block) + self.current_node += [msg_node] + return + # TODO check options are an un-nested dict / json serialize ? + elif content.startswith(":"): + content_lines = content.splitlines() # type: list + yaml_lines = [] + while content_lines: + if not content_lines[0].startswith(":"): + break + yaml_lines.append(content_lines.pop(0)[1:]) + yaml_block = "\n".join(yaml_lines) + content = "\n".join(content_lines) + try: + options = yaml.safe_load(yaml_block) or {} + except (yaml.parser.ParserError, yaml.scanner.ScannerError) as error: + msg_node = self.document.reporter.system_message( + 3, "Directive options:\n" + str(error), line=token.range[0] + ) # 3 is ERROR level + msg_node += nodes.literal_block(yaml_block, yaml_block) + self.current_node += [msg_node] + return + + # remove first line if blank + content_lines = content.splitlines() + if content_lines and not content_lines[0].strip(): + content_lines = content_lines[1:] # TODO directive name white/black lists directive_class, messages = directives.directive( @@ -466,13 +529,13 @@ def render_directive(self, token): # TODO option parsing options=options, # the directive content line by line - content=content.splitlines(), + content=content_lines, # the absolute line number of the first line of the directive lineno=token.range[0], # the line offset of the first line of the content content_offset=0, # a string containing the entire directive - block_text=content, + block_text="\n".join(content_lines), state=MockState(self, state_machine, token.range[0]), state_machine=state_machine, ) diff --git a/myst_parser/sphinx_parser.py b/myst_parser/sphinx_parser.py index a1291fb4..2558103e 100644 --- a/myst_parser/sphinx_parser.py +++ b/myst_parser/sphinx_parser.py @@ -169,4 +169,6 @@ def parse(self, inputstring, document): pass renderer = SphinxRenderer(document=document) with renderer: - renderer.render(Document(inputstring)) + # TODO capture parsing errors and report via docutils/sphinx + tokens = Document(inputstring) + renderer.render(tokens) diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..fe89c52e --- /dev/null +++ b/pytest.ini @@ -0,0 +1,4 @@ +[pytest] +addopts = --ignore=setup.py +markers = + sphinx: set parameters for the sphinx `app` fixture (see ipypublish/sphinx/tests/conftest.py) diff --git a/tests/sphinx_directives.json b/tests/sphinx_directives.json index 86f41086..ac7afa97 100644 --- a/tests/sphinx_directives.json +++ b/tests/sphinx_directives.json @@ -4,84 +4,96 @@ "class": "docutils.parsers.rst.directives.admonitions.Attention", "args": [], "options": {}, - "output": "" + "content": "a", + "output": "\n \n a" }, { "name": "caution", "class": "docutils.parsers.rst.directives.admonitions.Caution", "args": [], "options": {}, - "output": "" + "content": "a", + "output": "\n \n a" }, { "name": "danger", "class": "docutils.parsers.rst.directives.admonitions.Danger", "args": [], "options": {}, - "output": "" + "content": "a", + "output": "\n \n a" }, { "name": "error", "class": "docutils.parsers.rst.directives.admonitions.Error", "args": [], "options": {}, - "output": "" + "content": "a", + "output": "\n \n a" }, { "name": "important", "class": "docutils.parsers.rst.directives.admonitions.Important", "args": [], "options": {}, - "output": "" + "content": "a", + "output": "\n \n a" }, { "name": "note", "class": "docutils.parsers.rst.directives.admonitions.Note", "args": [], "options": {}, - "output": "" + "content": "a", + "output": "\n \n a" }, { "name": "tip", "class": "docutils.parsers.rst.directives.admonitions.Tip", "args": [], "options": {}, - "output": "" + "content": "a", + "output": "\n \n a" }, { "name": "hint", "class": "docutils.parsers.rst.directives.admonitions.Hint", "args": [], "options": {}, - "output": "" + "content": "a", + "output": "\n \n a" }, { "name": "warning", "class": "docutils.parsers.rst.directives.admonitions.Warning", "args": [], "options": {}, - "output": "" + "content": "a", + "output": "\n \n a" }, { "name": "admonition", "class": "docutils.parsers.rst.directives.admonitions.Admonition", "args": ["myclass"], "options": {}, - "output": "\n \n myclass" + "content": "a", + "output": "<admonition classes=\"admonition-myclass\">\n <title>\n myclass\n <paragraph>\n a" }, { "name": "sidebar", "class": "docutils.parsers.rst.directives.body.Sidebar", "args": ["sidebar title"], "options": {}, - "output": "<sidebar>\n <title>\n sidebar title" + "content": "a", + "output": "<sidebar>\n <title>\n sidebar title\n <paragraph>\n a" }, { "name": "topic", "class": "docutils.parsers.rst.directives.body.Topic", "args": ["Topic Title"], "options": {}, - "output": "<topic>\n <title>\n Topic Title" + "content": "a", + "output": "<topic>\n <title>\n Topic Title\n <paragraph>\n a" }, { "name": "line-block", @@ -95,7 +107,8 @@ "class": "docutils.parsers.rst.directives.body.ParsedLiteral", "args": [], "options": {}, - "output": "<literal_block xml:space=\"preserve\">" + "content": "a", + "output": "<literal_block xml:space=\"preserve\">\n a" }, { "name": "rubric", @@ -109,35 +122,41 @@ "class": "docutils.parsers.rst.directives.body.Epigraph", "args": [], "options": {}, - "output": "<block_quote classes=\"epigraph\">" + "content": "a\n\n-- attribution", + "output": "<block_quote classes=\"epigraph\">\n <paragraph>\n a\n <attribution>\n attribution" }, { "name": "highlights", "class": "docutils.parsers.rst.directives.body.Highlights", "args": [], "options": {}, - "output": "<block_quote classes=\"highlights\">" + "content": "a\n\n-- attribution", + "output": "<block_quote classes=\"highlights\">\n <paragraph>\n a\n <attribution>\n attribution" }, { "name": "pull-quote", + "content": "a\n\n-- attribution", "class": "docutils.parsers.rst.directives.body.PullQuote", "args": [], "options": {}, - "output": "<block_quote classes=\"pull-quote\">" + "content": "a\n\n-- attribution", + "output": "<block_quote classes=\"pull-quote\">\n <paragraph>\n a\n <attribution>\n attribution" }, { "name": "compound", "class": "docutils.parsers.rst.directives.body.Compound", "args": [], "options": {}, - "output": "<compound>" + "content": "a", + "output": "<compound>\n <paragraph>\n a" }, { "name": "container", "class": "docutils.parsers.rst.directives.body.Container", "args": [], "options": {}, - "output": "<container>" + "content": "a", + "output": "<container>\n <paragraph>\n a" }, { "name": "image", @@ -165,7 +184,8 @@ "class": "docutils.parsers.rst.directives.misc.Raw", "args": ["raw"], "options": {}, - "output": "<raw format=\"raw\" xml:space=\"preserve\">" + "content": "a", + "output": "<raw format=\"raw\" xml:space=\"preserve\">\n a" }, { "name": "replace", @@ -188,7 +208,8 @@ "class": "docutils.parsers.rst.directives.misc.Class", "args": ["myclass"], "options": {}, - "output": "" + "content": "a", + "output": "<paragraph classes=\"myclass\">\n a" }, { "name": "role", @@ -219,7 +240,7 @@ "class": "docutils.parsers.rst.directives.misc.TestDirective", "args": [], "options": {}, - "output": "<system_message level=\"1\" line=\"-1\" source=\"\" type=\"INFO\">\n <paragraph>\n Directive processed. Type=\"restructuredtext-test-directive\", arguments=[], options={}, content:\n <literal_block xml:space=\"preserve\">" + "output": "<system_message level=\"1\" line=\"0\" source=\"\" type=\"INFO\">\n <paragraph>\n Directive processed. Type=\"restructuredtext-test-directive\", arguments=[], options={}, content: None" }, { "name": "contents", @@ -240,14 +261,16 @@ "class": "docutils.parsers.rst.directives.parts.Header", "args": [], "options": {}, - "output": "<decoration>\n <header>" + "content": "a", + "output": "<decoration>\n <header>\n <paragraph>\n a" }, { "name": "footer", "class": "docutils.parsers.rst.directives.parts.Footer", "args": [], "options": {}, - "output": "<decoration>\n <footer>" + "content": "a", + "output": "<decoration>\n <footer>\n <paragraph>\n a" }, { "name": "target-notes", @@ -291,13 +314,6 @@ "options": {}, "output": "<highlightlang force=\"False\" lang=\"something\" linenothreshold=\"9223372036854775807\">" }, - { - "name": "highlightlang", - "class": "sphinx.directives.code.HighlightLang", - "args": ["something"], - "options": {}, - "output": "<highlightlang force=\"False\" lang=\"something\" linenothreshold=\"9223372036854775807\">" - }, { "name": "code-block", "class": "sphinx.directives.code.CodeBlock", @@ -318,7 +334,7 @@ "class": "sphinx.directives.code.LiteralInclude", "args": ["/path/to/file"], "options": {}, - "output": "<system_message level=\"2\" line=\"-1\" source=\"\" type=\"WARNING\">\n <paragraph>\n Include file '/srcdir/path/to/file' not found or reading it failed" + "output": "<system_message level=\"2\" line=\"0\" source=\"\" type=\"WARNING\">\n <paragraph>\n Include file '/srcdir/path/to/file' not found or reading it failed" }, { "name": "toctree", @@ -360,7 +376,8 @@ "class": "sphinx.directives.other.SeeAlso", "args": [], "options": {}, - "output": "<seealso>" + "content": "a", + "output": "<seealso>\n <paragraph>\n a" }, { "name": "tabularcolumns", @@ -449,7 +466,8 @@ "class": "sphinx.directives.patches.Code", "args": ["python"], "options": {}, - "output": "<literal_block force=\"False\" highlight_args=\"{}\" language=\"python\" xml:space=\"preserve\">" + "content": "a", + "output": "<literal_block force=\"False\" highlight_args=\"{}\" language=\"python\" xml:space=\"preserve\">\n a" }, { "name": "math", diff --git a/tests/test_docutils_renderer.py b/tests/test_docutils_renderer.py index 42970399..2d32406d 100644 --- a/tests/test_docutils_renderer.py +++ b/tests/test_docutils_renderer.py @@ -1,5 +1,6 @@ import json import os +import sys from textwrap import dedent, indent from unittest import mock @@ -354,18 +355,17 @@ def test_full_run(sphinx_renderer, file_regression): @pytest.mark.parametrize( - "role_data", + "name,role_data", [ - r + (r["name"], r) for r in roles_tests if r["import"].startswith("docutils") and not r["import"].endswith("unimplemented_role") and not r["import"].endswith("CustomRole") ], ) -def test_docutils_roles(renderer, role_data): +def test_docutils_roles(renderer, name, role_data): """""" - name = role_data["name"] if name in ["raw"]: # TODO fix skips pytest.skip("awaiting fix") @@ -384,18 +384,17 @@ def test_docutils_roles(renderer, role_data): @pytest.mark.parametrize( - "role_data", + "name,role_data", [ - r + (r["name"], r) for r in roles_tests if r["import"].startswith("sphinx") # and not r["import"].endswith("unimplemented_role") # and not r["import"].endswith("CustomRole") ], ) -def test_sphinx_roles(sphinx_renderer, role_data): +def test_sphinx_roles(sphinx_renderer, name, role_data): """""" - name = role_data["name"] # note, I think most of these have are actually node types rather than roles, # that I've erroneously picked up in my gather function. if name in [ @@ -455,17 +454,16 @@ def test_sphinx_roles(sphinx_renderer, role_data): @pytest.mark.parametrize( - "directive", + "name,directive", [ - d + (d["name"], d) for d in directive_tests if d["class"].startswith("docutils") and not d.get("sub_only", False) # todo add substitution definition directive and reference role ], ) -def test_docutils_directives(renderer, directive): +def test_docutils_directives(renderer, name, directive): """See https://docutils.sourceforge.io/docs/ref/rst/directives.html""" - name = directive["name"] if name in [ "role", "rst-class", @@ -497,16 +495,15 @@ def test_docutils_directives(renderer, directive): @pytest.mark.parametrize( - "directive", + "name,directive", [ - d + (d["name"], d) for d in directive_tests if d["class"].startswith("sphinx") and not d.get("sub_only", False) ], ) -def test_sphinx_directives(sphinx_renderer, directive): +def test_sphinx_directives(sphinx_renderer, name, directive): """See https://docutils.sourceforge.io/docs/ref/rst/directives.html""" - name = directive["name"] if name in ["csv-table", "meta", "include"]: # TODO fix skips pytest.skip("awaiting fix") @@ -531,3 +528,62 @@ def test_sphinx_directives(sphinx_renderer, directive): + indent(directive["output"], " ") + ("\n" if directive["output"] else "") ) + + +@pytest.mark.skipif( + sys.version_info.major == 3 and sys.version_info.minor <= 5, + reason="option dict keys in wrong order", +) +@pytest.mark.parametrize( + "type,text", + [ + ( + "block_style", + ("---", "option1: a", "option2: b", "---", "", "content", "```"), + ), + ("colon_style", (":option1: a", ":option2: b", "", "content", "```")), + ( + "block_style_no_space", + ("---", "option1: a", "option2: b", "---", "content", "```"), + ), + ("colon_style_no_space", (":option1: a", ":option2: b", "content", "```")), + ], +) +def test_directive_options(renderer, type, text): + renderer.render(Document(["```{restructuredtext-test-directive}"] + list(text))) + assert renderer.document.pformat() == dedent( + """\ + <document source=""> + <system_message level="1" line="0" source="" type="INFO"> + <paragraph> + Directive processed. Type="restructuredtext-test-directive", arguments=[], options={'option1': 'a', 'option2': 'b'}, content: + <literal_block xml:space="preserve"> + content + """ # noqa: E501 + ) + + +@pytest.mark.parametrize( + "type,text", + [ + ("block_style", ("---", "option1", "option2: b", "---", "", "content", "```")), + ("colon_style", (":option1", ":option2: b", "", "content", "```")), + ], +) +def test_directive_options_error(renderer, type, text): + renderer.render(Document(["```{restructuredtext-test-directive}"] + list(text))) + assert renderer.document.pformat() == dedent( + """\ + <document source=""> + <system_message level="3" line="0" source="" type="ERROR"> + <paragraph> + Directive options: + mapping values are not allowed here + in "<unicode string>", line 2, column 8: + option2: b + ^ + <literal_block xml:space="preserve"> + option1 + option2: b + """ + )