👌‼️ Allow meta_html/substitutions in docutils (#672)

Refactors `myst_parser/parsers/docutils_.py` slightly,
by removing `DOCUTILS_EXCLUDED_ARGS`,
and removing the `excluded` argument from `create_myst_settings_spec` and `create_myst_config`.

.github/workflows/tests.yml

@@ -86,7 +86,7 @@ jobs:
       run: python .github/workflows/docutils_setup.py pyproject.toml README.md
     - name: Install dependencies
       run: |
-        pip install . pytest~=6.2 pytest-param-files~=0.3.3 pygments docutils==${{ matrix.docutils-version }}
+        pip install .[linkify,testing-docutils] docutils==${{ matrix.docutils-version }}
     - name: ensure sphinx is not installed
       run: |
         python -c "\
@@ -97,7 +97,7 @@ jobs:
         else:
             raise AssertionError()"
     - name: Run pytest for docutils-only tests
-      run: pytest tests/test_docutils.py tests/test_renderers/test_fixtures_docutils.py tests/test_renderers/test_include_directive.py
+      run: pytest tests/test_docutils.py tests/test_renderers/test_fixtures_docutils.py tests/test_renderers/test_include_directive.py tests/test_renderers/test_myst_config.py
     - name: Run docutils CLI
       run: echo "test" | myst-docutils-html
 

docs/docutils.md

@@ -46,6 +46,9 @@ The CLI commands can also utilise the [`docutils.conf` configuration file](https
 [general]
 myst-enable-extensions: deflist,linkify
 myst-footnote-transition: no
+myst-substitutions:
+    key1: value1
+    key2: value2
 
 # These entries affect specific HTML output:
 [html writers]

myst_parser/_docs.py

@@ -77,7 +77,7 @@ def run(self):
                 continue
 
             # filter by sphinx options
-            if "sphinx" in self.options and field.metadata.get("sphinx_exclude"):
+            if "sphinx" in self.options and field.metadata.get("docutils_only"):
                 continue
 
             if "extensions" in self.options:

myst_parser/config/dc_validators.py

@@ -38,7 +38,7 @@ def validate_fields(inst: Any) -> None:
 
 class ValidatorType(Protocol):
     def __call__(
-        self, inst: bytes, field: dc.Field, value: Any, suffix: str = ""
+        self, inst: Any, field: dc.Field, value: Any, suffix: str = ""
     ) -> None:
         ...
 

myst_parser/config/main.py

@@ -26,9 +26,10 @@
 )
 
 
-def check_extensions(_, __, value):
+def check_extensions(_, field: dc.Field, value: Any):
+    """Check that the extensions are a list of known strings"""
     if not isinstance(value, Iterable):
-        raise TypeError(f"'enable_extensions' not iterable: {value}")
+        raise TypeError(f"'{field.name}' not iterable: {value}")
     diff = set(value).difference(
         [
             "amsmath",
@@ -49,16 +50,17 @@ def check_extensions(_, __, value):
         ]
     )
     if diff:
-        raise ValueError(f"'enable_extensions' items not recognised: {diff}")
+        raise ValueError(f"'{field.name}' items not recognised: {diff}")
 
 
-def check_sub_delimiters(_, __, value):
+def check_sub_delimiters(_, field: dc.Field, value: Any):
+    """Check that the sub_delimiters are a tuple of length 2 of strings of length 1"""
     if (not isinstance(value, (tuple, list))) or len(value) != 2:
-        raise TypeError(f"myst_sub_delimiters is not a tuple of length 2: {value}")
+        raise TypeError(f"'{field.name}' is not a tuple of length 2: {value}")
     for delim in value:
         if (not isinstance(delim, str)) or len(delim) != 1:
             raise TypeError(
-                f"myst_sub_delimiters does not contain strings of length 1: {value}"
+                f"'{field.name}' does not contain strings of length 1: {value}"
             )
 
 
@@ -125,6 +127,7 @@ class MdParserConfig:
                 deep_iterable(instance_of(str), instance_of((list, tuple)))
             ),
             "help": "Sphinx domain names to search in for link references",
+            "sphinx_only": True,
         },
     )
 
@@ -149,6 +152,7 @@ class MdParserConfig:
         metadata={
             "validator": optional(in_([1, 2, 3, 4, 5, 6, 7])),
             "help": "Heading level depth to assign HTML anchors",
+            "sphinx_only": True,
         },
     )
 
@@ -158,6 +162,7 @@ class MdParserConfig:
             "validator": optional(is_callable),
             "help": "Function for creating heading anchors",
             "global_only": True,
+            "sphinx_only": True,
         },
     )
 
@@ -210,6 +215,7 @@ class MdParserConfig:
             "validator": check_sub_delimiters,
             "help": "Substitution delimiters",
             "extension": "substitutions",
+            "sphinx_only": True,
         },
     )
 
@@ -262,6 +268,7 @@ class MdParserConfig:
             "help": "Update sphinx.ext.mathjax configuration to ignore `$` delimiters",
             "extension": "dollarmath",
             "global_only": True,
+            "sphinx_only": True,
         },
     )
 
@@ -272,6 +279,7 @@ class MdParserConfig:
             "help": "MathJax classes to add to math HTML",
             "extension": "dollarmath",
             "global_only": True,
+            "sphinx_only": True,
         },
     )
 

myst_parser/mdit_to_docutils/base.py

@@ -1461,16 +1461,12 @@ def html_meta_to_nodes(
         return []
 
     try:
-        # if sphinx available
-        from sphinx.addnodes import meta as meta_cls
-    except ImportError:
-        try:
-            # docutils >= 0.19
-            meta_cls = nodes.meta  # type: ignore
-        except AttributeError:
-            from docutils.parsers.rst.directives.html import MetaBody
+        meta_cls = nodes.meta
+    except AttributeError:
+        # docutils-0.17 or older
+        from docutils.parsers.rst.directives.html import MetaBody
 
-            meta_cls = MetaBody.meta  # type: ignore
+        meta_cls = MetaBody.meta
 
     output = []
 

myst_parser/parsers/docutils_.py

@@ -2,6 +2,7 @@
 from dataclasses import Field
 from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple, Union
 
+import yaml
 from docutils import frontend, nodes
 from docutils.core import default_description, publish_cmdline
 from docutils.parsers.rst import Parser as RstParser
@@ -58,32 +59,39 @@ def __bool__(self):
 """Sentinel for arguments not set through docutils.conf."""
 
 
-DOCUTILS_EXCLUDED_ARGS = (
-    # docutils.conf can't represent callables
-    "heading_slug_func",
-    # docutils.conf can't represent dicts
-    "html_meta",
-    "substitutions",
-    # we can't add substitutions so not needed
-    "sub_delimiters",
-    # sphinx only options
-    "heading_anchors",
-    "ref_domains",
-    "update_mathjax",
-    "mathjax_classes",
-)
-"""Names of settings that cannot be set in docutils.conf."""
+def _create_validate_yaml(field: Field):
+    """Create a deserializer/validator for a json setting."""
+
+    def _validate_yaml(
+        setting, value, option_parser, config_parser=None, config_section=None
+    ):
+        """Check/normalize a key-value pair setting.
+
+        Items delimited by `,`, and key-value pairs delimited by `=`.
+        """
+        try:
+            output = yaml.safe_load(value)
+        except Exception:
+            raise ValueError("Invalid YAML string")
+        if "validator" in field.metadata:
+            field.metadata["validator"](None, field, output)
+        return output
+
+    return _validate_yaml
 
 
 def _attr_to_optparse_option(at: Field, default: Any) -> Tuple[dict, str]:
-    """Convert a field into a Docutils optparse options dict."""
+    """Convert a field into a Docutils optparse options dict.
+
+    :returns: (option_dict, default)
+    """
     if at.type is int:
-        return {"metavar": "<int>", "validator": _validate_int}, f"(default: {default})"
+        return {"metavar": "<int>", "validator": _validate_int}, str(default)
     if at.type is bool:
         return {
             "metavar": "<boolean>",
             "validator": frontend.validate_boolean,
-        }, f"(default: {default})"
+        }, str(default)
     if at.type is str:
         return {
             "metavar": "<str>",
@@ -96,28 +104,32 @@ def _attr_to_optparse_option(at: Field, default: Any) -> Tuple[dict, str]:
             "metavar": f"<{'|'.join(repr(a) for a in args)}>",
             "type": "choice",
             "choices": args,
-        }, f"(default: {default!r})"
+        }, repr(default)
     if at.type in (Iterable[str], Sequence[str]):
         return {
             "metavar": "<comma-delimited>",
             "validator": frontend.validate_comma_separated_list,
-        }, f"(default: '{','.join(default)}')"
+        }, ",".join(default)
     if at.type == Tuple[str, str]:
         return {
             "metavar": "<str,str>",
             "validator": _create_validate_tuple(2),
-        }, f"(default: '{','.join(default)}')"
+        }, ",".join(default)
     if at.type == Union[int, type(None)]:
         return {
             "metavar": "<null|int>",
             "validator": _validate_int,
-        }, f"(default: {default})"
+        }, str(default)
     if at.type == Union[Iterable[str], type(None)]:
-        default_str = ",".join(default) if default else ""
         return {
             "metavar": "<null|comma-delimited>",
             "validator": frontend.validate_comma_separated_list,
-        }, f"(default: {default_str!r})"
+        }, ",".join(default) if default else ""
+    if get_origin(at.type) is dict:
+        return {
+            "metavar": "<yaml-dict>",
+            "validator": _create_validate_yaml(at),
+        }, str(default) if default else ""
     raise AssertionError(
         f"Configuration option {at.name} not set up for use in docutils.conf."
     )
@@ -133,34 +145,33 @@ def attr_to_optparse_option(
     name = f"{prefix}{attribute.name}"
     flag = "--" + name.replace("_", "-")
     options = {"dest": name, "default": DOCUTILS_UNSET}
-    at_options, type_str = _attr_to_optparse_option(attribute, default)
+    at_options, default_str = _attr_to_optparse_option(attribute, default)
     options.update(at_options)
     help_str = attribute.metadata.get("help", "") if attribute.metadata else ""
-    return (f"{help_str} {type_str}", [flag], options)
+    if default_str:
+        help_str += f" (default: {default_str})"
+    return (help_str, [flag], options)
 
 
-def create_myst_settings_spec(
-    excluded: Sequence[str], config_cls=MdParserConfig, prefix: str = "myst_"
-):
+def create_myst_settings_spec(config_cls=MdParserConfig, prefix: str = "myst_"):
     """Return a list of Docutils setting for the docutils MyST section."""
     defaults = config_cls()
     return tuple(
         attr_to_optparse_option(at, getattr(defaults, at.name), prefix)
         for at in config_cls.get_fields()
-        if at.name not in excluded
+        if (not at.metadata.get("sphinx_only", False))
     )
 
 
 def create_myst_config(
     settings: frontend.Values,
-    excluded: Sequence[str],
     config_cls=MdParserConfig,
     prefix: str = "myst_",
 ):
     """Create a configuration instance from the given settings."""
     values = {}
     for attribute in config_cls.get_fields():
-        if attribute.name in excluded:
+        if attribute.metadata.get("sphinx_only", False):
             continue
         setting = f"{prefix}{attribute.name}"
         val = getattr(settings, setting, DOCUTILS_UNSET)
@@ -178,7 +189,7 @@ class Parser(RstParser):
     settings_spec = (
         "MyST options",
         None,
-        create_myst_settings_spec(DOCUTILS_EXCLUDED_ARGS),
+        create_myst_settings_spec(),
         *RstParser.settings_spec,
     )
     """Runtime settings specification."""
@@ -209,7 +220,7 @@ def parse(self, inputstring: str, document: nodes.document) -> None:
 
         # create parsing configuration from the global config
         try:
-            config = create_myst_config(document.settings, DOCUTILS_EXCLUDED_ARGS)
+            config = create_myst_config(document.settings)
         except Exception as exc:
             error = document.reporter.error(f"Global myst configuration invalid: {exc}")
             document.append(error)

pyproject.toml

@@ -70,6 +70,11 @@ testing = [
     "pytest-param-files~=0.3.4",
     "sphinx-pytest",
 ]
+testing-docutils = [
+    "pygments",
+    "pytest>=6,<7",
+    "pytest-param-files~=0.3.4",
+]
 
 [project.scripts]
 myst-anchors = "myst_parser.cli:print_anchors"

tests/test_renderers/fixtures/myst-config.txt

@@ -156,6 +156,27 @@ www.commonmark.org/he<lp
         <lp
 .
 
+[html_meta] --myst-html-meta='{"keywords": "Sphinx, MyST"}'
+.
+text
+.
+<document source="<string>">
+    <meta content="Sphinx, MyST" name="keywords">
+    <paragraph>
+        text
+.
+
+[substitutions] --myst-enable-extensions=substitution --myst-substitutions='{"a": "b", "c": "d"}'
+.
+{{a}} {{c}}
+.
+<document source="<string>">
+    <paragraph>
+        b
+
+        d
+.
+
 [attrs_inline_span] --myst-enable-extensions=attrs_inline
 .
 [content]{#id .a .b}

tests/test_renderers/test_myst_config.py

@@ -4,7 +4,7 @@
 from pathlib import Path
 
 import pytest
-from docutils.core import Publisher, publish_doctree
+from docutils.core import Publisher, publish_string
 
 from myst_parser.parsers.docutils_ import Parser
 
@@ -25,13 +25,14 @@ def test_cmdline(file_params):
             f"Failed to parse commandline: {file_params.description}\n{err}"
         )
     report_stream = StringIO()
+    settings["output_encoding"] = "unicode"
     settings["warning_stream"] = report_stream
-    doctree = publish_doctree(
+    output = publish_string(
         file_params.content,
         parser=Parser(),
+        writer_name="pseudoxml",
         settings_overrides=settings,
     )
-    output = doctree.pformat()
     warnings = report_stream.getvalue()
     if warnings:
         output += "\n" + warnings