cli, docutils, pygments: Generate Pygments stylesheet dynamically

Closes GH-58.

* alectryon/cli.py (gen_docutils): Skip embedded assets.
(copy_assets): Add support for dynamically-generated assets.
(dump_html_standalone): Don't embed pygments stylesheet.
(build_parser): Add a new --pygments-stylesheet option.
* alectryon/docutils.py (register_stylesheets): Support dynamic assets.
* alectryon/html.py (ASSETS): Change "tango_subtle" to "pygments".
* alectryon/latex.py (ASSETS): Same.

Suggested-by: @gares

alectryon/cli.py

@@ -18,6 +18,8 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
+from typing import Tuple, List, Union
+
 import argparse
 import inspect
 import os
@@ -93,6 +95,7 @@ def register_docutils(v, ctx):
         'input_encoding': 'utf-8',
         'output_encoding': 'utf-8',
         'exit_status_level': 3,
+        'pygments_style': ctx["pygments_style"],
         'alectryon_banner': ctx["include_banner"],
         'alectryon_vernums': ctx["include_vernums"],
         'alectryon_webpage_style': ctx["webpage_style"],
@@ -122,7 +125,7 @@ def _gen_docutils(source, fpath,
 
     max_level = pub.document.reporter.max_level
     exit_code = max_level + 10 if max_level >= pub.settings.exit_status_level else 0
-    return output.decode("utf-8"), exit_code
+    return output.decode("utf-8"), pub, exit_code
 
 def _resolve_dialect(backend, html_dialect, latex_dialect):
     return {"webpage": html_dialect, "latex": latex_dialect}.get(backend, None)
@@ -133,15 +136,19 @@ def _record_assets(assets, path, names):
 
 def gen_docutils(src, frontend, backend, fpath, dialect,
                  docutils_settings_overrides, assets, exit_code):
-    from .docutils import get_pipeline
+    from .docutils import get_pipeline, alectryon_state
 
     pipeline = get_pipeline(frontend, backend, dialect)
-    _record_assets(assets, pipeline.translator.ASSETS_PATH, pipeline.translator.ASSETS)
-
-    output, exit_code.val = \
+    output, pub, exit_code.val = \
         _gen_docutils(src, fpath,
                       pipeline.parser, pipeline.reader, pipeline.writer,
                       docutils_settings_overrides)
+
+    embedded_assets = alectryon_state(pub.document).embedded_assets
+    _record_assets(assets,
+                   pipeline.translator.ASSETS_PATH,
+                   [a for a in pipeline.translator.ASSETS if a not in embedded_assets])
+
     return output
 
 def _docutils_cmdline(description, frontend, backend):
@@ -242,13 +249,20 @@ def gen_html_snippets_with_coqdoc(annotated, html_classes, fname, html_minificat
     # ‘return’ instead of ‘yield from’ to update html_classes eagerly
     return _gen_html_snippets_with_coqdoc(annotated, fname, html_minification)
 
-def copy_assets(state, assets, copy_fn, output_directory):
+def copy_assets(state, assets: List[Tuple[str, Union[str, core.Asset]]],
+                copy_fn, output_directory, ctx):
     if copy_fn is None:
         return state
 
-    for (path, name) in assets:
-        src = os.path.join(path, name)
-        dst = os.path.join(output_directory, name)
+    for (path, asset) in assets:
+        src = os.path.join(path, asset)
+        dst = os.path.join(output_directory, asset)
+
+        if isinstance(asset, core.Asset):
+            with open(dst, mode="w") as f:
+                f.write(asset.gen(ctx))
+            continue
+
         if copy_fn is not shutil.copyfile:
             try:
                 os.unlink(dst)
@@ -268,7 +282,6 @@ def dump_html_standalone(snippets, fname, webpage_style,
     from dominate.util import raw
     from . import GENERATOR
     from .core import SerAPI
-    from .pygments import HTML_FORMATTER
     from .html import ASSETS, ADDITIONAL_HEADS, JS_UNMINIFY, gen_banner, wrap_classes
 
     doc = document(title=fname)
@@ -281,18 +294,16 @@ def dump_html_standalone(snippets, fname, webpage_style,
         doc.head.add(raw(hd))
     if html_minification:
         doc.head.add(raw(JS_UNMINIFY))
-    for css in ASSETS.ALECTRYON_CSS:
-        doc.head.add(tags.link(rel="stylesheet", href=css))
+    for css in ASSETS.ALECTRYON_CSS + ASSETS.PYGMENTS_CSS:
+        doc.head.add(tags.link(rel="stylesheet", href=str(css)))
     for link in (ASSETS.IBM_PLEX_CDN, ASSETS.FIRA_CODE_CDN):
-        doc.head.add(raw(link))
+        doc.head.add(raw("\n    " + link))
     for js in ASSETS.ALECTRYON_JS:
         doc.head.add(tags.script(src=js))
 
     _record_assets(assets, ASSETS.PATH, ASSETS.ALECTRYON_CSS)
     _record_assets(assets, ASSETS.PATH, ASSETS.ALECTRYON_JS)
-
-    pygments_css = HTML_FORMATTER.get_style_defs('.highlight')
-    doc.head.add(tags.style(pygments_css, type="text/css"))
+    _record_assets(assets, ASSETS.PATH, ASSETS.PYGMENTS_CSS)
 
     if html_minification:
         html_classes.append("minified")
@@ -571,12 +582,16 @@ def build_parser():
     out.add_argument("--output-directory", default=None,
                      help=OUT_DIR_HELP)
 
-    COPY_ASSETS_HELP = ("Chose the method to use to copy assets " +
-                        "along the generated file(s) when creating webpages.")
+    COPY_ASSETS_HELP = ("Chose the method to use to copy assets along the " +
+                        "generated file(s) when creating webpages and TeX docs.")
     out.add_argument("--copy-assets", choices=list(COPY_FUNCTIONS.keys()),
                      default="copy", dest="copy_fn",
                      help=COPY_ASSETS_HELP)
 
+    PYGMENTS_STYLE_HELP = "Choose a pygments style by name."
+    out.add_argument("--pygments-style", default=None,
+                     help=PYGMENTS_STYLE_HELP)
+
     MARK_POINT_HELP = "Mark a point in the output with a given marker."
     out.add_argument("--mark-point", nargs=2, default=(None, None),
                      metavar=("POINT", "MARKER"),

alectryon/core.py

@@ -126,6 +126,14 @@ def _gen_any(self, obj):
         else:
             raise TypeError("Unexpected object type: {}".format(type(obj)))
 
+class Asset(str):
+    def __new__(cls, fname, _gen):
+        return super().__new__(cls, fname)
+
+    def __init__(self, _fname, gen):
+        super().__init__()
+        self.gen = gen
+
 class Position(namedtuple("Position", "fpath line col")):
     def as_header(self):
         return "{}:{}:{}:".format(self.fpath or "<unknown>", self.line, self.col)

alectryon/docutils.py

@@ -89,7 +89,7 @@
 
 from . import core, transforms, html, latex, markers
 from .core import Gensym, SerAPI, Position, PosStr
-from .pygments import highlight_html, highlight_latex, added_tokens, \
+from .pygments import make_highlighter, added_tokens, validate_style, \
     resolve_token, replace_builtin_coq_lexer
 
 # reST extensions
@@ -165,6 +165,7 @@ def __init__(self, document):
         self.generator: Optional[core.GeneratorInfo] = None
         self.root_language: Optional[str] = None
         self.transforms_executed = set()
+        self.embedded_assets = []
         self.document = document
         self._config = None
 
@@ -178,12 +179,20 @@ def populate_config(self):
     def config(self):
         return self.populate_config()
 
-def _alectryon_state(document):
+def alectryon_state(document):
     st = getattr(document, "alectryon_state", None)
     if st is None:
         st = document.alectryon_state = AlectryonState(document)
     return st
 
+def _docutils_config(document, attr, default=None):
+    """Look up `attr` in Sphinx config, falling back to docutils settings."""
+    settings = document.settings
+    value = getattr(settings, attr, default)
+    if hasattr(settings, "env"):
+        value = getattr(settings.env.config, attr, value)
+    return value
+
 def _gensym_stem(document, suffix=""):
     source = document.get('source', "")
     return nodes.make_id(os.path.basename(source)) + (source and suffix)
@@ -256,7 +265,7 @@ def apply(self, **_kwargs):
         # directive, and potentially once by add_transform in Sphinx, so we need
         # to make sure that running them twice is safe (in particular, we must
         # not overwrite the cache).
-        state = _alectryon_state(self.document)
+        state = alectryon_state(self.document)
         if type(self).__name__ not in state.transforms_executed:
             state.transforms_executed.add(type(self).__name__)
             self._apply()
@@ -271,7 +280,7 @@ class LoadConfigTransform(OneTimeTransform):
     default_priority = 300
 
     def _apply(self):
-        _alectryon_state(self.document).populate_config()
+        alectryon_state(self.document).populate_config()
 
 class ActivateMathJaxTransform(Transform):
     """Add the ``mathjax_process`` class on math nodes.
@@ -336,7 +345,7 @@ def annotate_cached(self, chunks, sertop_args):
         return cache.generator, annotated
 
     def annotate(self, pending_nodes):
-        config = _alectryon_state(self.document).config
+        config = alectryon_state(self.document).config
         sertop_args = (*self.SERTOP_ARGS, *config.sertop_args)
         chunks = [pending.details["contents"] for pending in pending_nodes]
         return self.annotate_cached(chunks, sertop_args)
@@ -357,7 +366,7 @@ def replace_node(self, pending, fragments):
     def apply_coq(self):
         pending_nodes = list(self.document.traverse(alectryon_pending))
         generator, annotated = self.annotate(pending_nodes)
-        _alectryon_state(self.document).generator = generator
+        alectryon_state(self.document).generator = generator
         for node, fragments in zip(pending_nodes, annotated):
             self._try(self.replace_node, node, fragments)
 
@@ -547,11 +556,14 @@ class AlectryonPostTransform(OneTimeTransform):
 
     def init_generator(self):
         formats = set(self.document.transformer.components['writer'].supported)
+        style = _docutils_config(self.document, "pygments_style")
         if 'html' in formats:
+            highlighter = make_highlighter("html", style)
             return "html", html.HtmlGenerator(
-                highlight_html, _gensym_stem(self.document), HTML_MINIFICATION)
+                highlighter, _gensym_stem(self.document), HTML_MINIFICATION)
         if {'latex', 'xelatex', 'lualatex'} & formats:
-            return "latex", latex.LatexGenerator(highlight_latex)
+            highlighter = make_highlighter("latex", style)
+            return "latex", latex.LatexGenerator(highlighter)
         raise NotImplementedError("Unknown output format")
 
     @staticmethod
@@ -571,7 +583,7 @@ def replace_one_quote(node, fmt, generator):
 
     def _apply(self, **_kwargs):
         fmt, generator = self.init_generator()
-        with added_tokens(_alectryon_state(self.document).config.tokens):
+        with added_tokens(alectryon_state(self.document).config.tokens):
             for node in self.document.traverse(alectryon_pending_io):
                 self.replace_one_io(node, fmt, generator)
             for node in self.document.traverse(alectryon_pending_quote):
@@ -669,7 +681,7 @@ def run(self):
 
         col_offset = indent
         if document.get('source', "") == source \
-           and _alectryon_state(document).root_language == "coq":
+           and alectryon_state(document).root_language == "coq":
             col_offset = 0
 
         pos = Position(source, line, col_offset)
@@ -976,7 +988,7 @@ def parse(self, inputstring, document):
         from .literate import ParsingError
         self.setup_parse(inputstring, document)
         # pylint: disable=attribute-defined-outside-init
-        _alectryon_state(document).root_language = "coq"
+        alectryon_state(document).root_language = "coq"
         self.statemachine = docutils.parsers.rst.states.RSTStateMachine( # type: ignore
             state_classes=self.state_classes,
             initial_state=self.initial_state,
@@ -993,11 +1005,17 @@ def parse(self, inputstring, document):
 # ------
 
 def register_stylesheets(translator, stylesheets, assets_path):
-    for name in stylesheets:
+    for asset in stylesheets:
         if translator.settings.embed_stylesheet:
+            alectryon_state(translator.document).embedded_assets.append(asset)
+            if isinstance(asset, core.Asset):
+                # Inline by hand, since the file doesn't exist on disk
+                contents = asset.gen(vars(translator.settings))
+                translator.stylesheet.append(translator.embedded_stylesheet % contents)
+                continue
             # Expand only if we're going to inline; otherwise keep relative
-            name = os.path.join(assets_path, name)
-        translator.stylesheet.append(translator.stylesheet_call(name))
+            asset = os.path.join(assets_path, asset)
+        translator.stylesheet.append(translator.stylesheet_call(asset))
 
 def make_HtmlTranslator(base):
     class Translator(base):
@@ -1037,7 +1055,7 @@ def __init__(self, document):
             self.body_prefix.append('<div class="{}">'.format(cls))
 
             if self.settings.alectryon_banner:
-                generator = _alectryon_state(document).generator
+                generator = alectryon_state(document).generator
                 include_vernums = document.settings.alectryon_vernums
                 self.body_prefix.append(html.gen_banner(generator, include_vernums))
 
@@ -1055,6 +1073,10 @@ def __init__(self, document):
      {"choices": ("centered", "floating", "windowed"),
       "dest": "alectryon_webpage_style",
       "default": "centered", "metavar": "STYLE"}),
+    ("Choose a Pygments style by name",
+     ["--pygments-style"],
+     {'default': None, 'dest': "pygments_style",
+      'validator': validate_style}),
     ("Omit Alectryon's explanatory header",
      ["--no-header"],
      {'default': True, 'action': 'store_false',
@@ -1095,6 +1117,8 @@ class Translator(base):
         ASSETS = STY
         ASSETS_PATH = latex.ASSETS.PATH
 
+        embedded_stylesheet = "%% embedded stylesheet\n\\makeatletter\n%s\n\\makeatother\n"
+
         def __init__(self, document, *args, **kwargs):
             super().__init__(document, *args, **kwargs)
             register_stylesheets(self, self.STY, self.ASSETS_PATH)

alectryon/html.py

@@ -27,7 +27,7 @@
 from dominate.util import text as txt
 
 from . import transforms, GENERATOR
-from .core import b16, Gensym, Backend, Text, RichSentence, Goals, Messages
+from .core import b16, Gensym, Backend, Text, RichSentence, Goals, Messages, Asset
 
 _SELF_PATH = path.dirname(path.realpath(__file__))
 
@@ -38,10 +38,18 @@
 class ASSETS:
     PATH = path.join(_SELF_PATH, "assets")
 
+    @staticmethod
+    def gen_css(ctx):
+        from .pygments import get_stylesheet
+        style = ctx.get("pygments_style", None)
+        BANNER = "/* Pygments stylesheet generated by Alectryon (style={}) */\n"
+        return BANNER.format(style) + get_stylesheet("html", style)
+    gen_css.fname = "pygments.css" # type: ignore
+
     ALECTRYON_CSS = ("alectryon.css",)
     ALECTRYON_JS = ("alectryon.js",)
 
-    PYGMENTS_CSS = ("tango_subtle.min.css",)
+    PYGMENTS_CSS = (Asset("pygments.css", gen_css.__func__),) # type: ignore
     DOCUTILS_CSS = ("docutils_basic.css",)
 
     IBM_PLEX_CDN = '<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/IBM-type/0.5.4/css/ibm-type.min.css" integrity="sha512-sky5cf9Ts6FY1kstGOBHSybfKqdHR41M0Ldb0BjNiv3ifltoQIsg0zIaQ+wwdwgQ0w9vKFW7Js50lxH9vqNSSw==" crossorigin="anonymous" />' # pylint: disable=line-too-long

alectryon/latex.py

@@ -23,15 +23,22 @@
 import re
 from os import path
 
-from .core import Backend, Text, RichSentence, Messages, Goals
+from .core import Backend, Text, RichSentence, Messages, Goals, Asset
 from . import transforms, GENERATOR
 
 _SELF_PATH = path.dirname(path.realpath(__file__))
 
 class ASSETS:
     PATH = path.join(_SELF_PATH, "assets")
 
-    PYGMENTS_STY = ("tango_subtle.sty",)
+    @staticmethod
+    def gen_sty(ctx):
+        from .pygments import get_stylesheet
+        style = ctx.get("pygments_style", None)
+        BANNER = "% Pygments stylesheet generated by Alectryon (style={})\n"
+        return BANNER.format(style) + get_stylesheet("latex", style)
+
+    PYGMENTS_STY = (Asset("pygments.sty", gen_sty.__func__),) # type: ignore
     ALECTRYON_STY = ("alectryon.sty",)
 
 def format_macro(name, args, optargs, before_optargs=None):