diff --git a/nixos/lib/make-options-doc/default.nix b/nixos/lib/make-options-doc/default.nix index a0b9136ca7bc6c..a2385582a014da 100644 --- a/nixos/lib/make-options-doc/default.nix +++ b/nixos/lib/make-options-doc/default.nix @@ -91,11 +91,14 @@ let in rec { inherit optionsNix; - optionsAsciiDoc = pkgs.runCommand "options.adoc" {} '' - ${pkgs.python3Minimal}/bin/python ${./generateDoc.py} \ - --format asciidoc \ + optionsAsciiDoc = pkgs.runCommand "options.adoc" { + nativeBuildInputs = [ pkgs.nixos-render-docs ]; + } '' + nixos-render-docs -j $NIX_BUILD_CORES options asciidoc \ + --manpage-urls ${pkgs.path + "/doc/manpage-urls.json"} \ + --revision ${lib.escapeShellArg revision} \ ${optionsJSON}/share/doc/nixos/options.json \ - > $out + $out ''; optionsCommonMark = pkgs.runCommand "options.md" { diff --git a/nixos/lib/make-options-doc/generateDoc.py b/nixos/lib/make-options-doc/generateDoc.py deleted file mode 100644 index a41255067bfba6..00000000000000 --- a/nixos/lib/make-options-doc/generateDoc.py +++ /dev/null @@ -1,83 +0,0 @@ -import argparse -import json -import sys - -formats = ['asciidoc'] - -parser = argparse.ArgumentParser( - description = 'Generate documentation for a set of JSON-formatted NixOS options' -) -parser.add_argument( - 'nix_options_path', - help = 'a path to a JSON file containing the NixOS options' -) -parser.add_argument( - '-f', - '--format', - choices = formats, - required = True, - help = f'the documentation format to generate' -) - -args = parser.parse_args() - -class OptionsEncoder(json.JSONEncoder): - def encode(self, obj): - # Unpack literal expressions and other Nix types. - # Don't escape the strings: they were escaped when initially serialized to JSON. - if isinstance(obj, dict): - _type = obj.get('_type') - if _type is not None: - if _type == 'literalExpression' or _type == 'literalDocBook': - return obj['text'] - - if _type == 'derivation': - return obj['name'] - - raise Exception(f'Unexpected type `{_type}` in {json.dumps(obj)}') - - return super().encode(obj) - -# TODO: declarations: link to github -def generate_asciidoc(options): - for (name, value) in options.items(): - print(f'== {name}') - print() - print(value['description']) - print() - print('[discrete]') - print('=== details') - print() - print(f'Type:: {value["type"]}') - if 'default' in value: - print('Default::') - print('+') - print('----') - print(json.dumps(value['default'], cls=OptionsEncoder, ensure_ascii=False, separators=(',', ':'))) - print('----') - print() - else: - print('No Default:: {blank}') - if value['readOnly']: - print('Read Only:: {blank}') - else: - print() - if 'example' in value: - print('Example::') - print('+') - print('----') - print(json.dumps(value['example'], cls=OptionsEncoder, ensure_ascii=False, separators=(',', ':'))) - print('----') - print() - else: - print('No Example:: {blank}') - print() - -with open(args.nix_options_path) as nix_options_json: - options = json.load(nix_options_json) - - if args.format == 'asciidoc': - generate_asciidoc(options) - else: - raise Exception(f'Unsupported documentation format `--format {args.format}`') - diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/asciidoc.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/asciidoc.py new file mode 100644 index 00000000000000..637185227e83f4 --- /dev/null +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/asciidoc.py @@ -0,0 +1,262 @@ +from collections.abc import Mapping, MutableMapping, Sequence +from dataclasses import dataclass +from typing import Any, cast, Optional +from urllib.parse import quote + +from .md import Renderer + +import markdown_it +from markdown_it.token import Token +from markdown_it.utils import OptionsDict + +_asciidoc_escapes = { + # escape all dots, just in case one is pasted at SOL + ord('.'): "{zwsp}.", + # may be replaced by typographic variants + ord("'"): "{apos}", + ord('"'): "{quot}", + # passthrough character + ord('+'): "{plus}", + # table marker + ord('|'): "{vbar}", + # xml entity reference + ord('&'): "{amp}", + # crossrefs. < needs extra escaping because links break in odd ways if they start with it + ord('<'): "{zwsp}+<+{zwsp}", + ord('>'): "{gt}", + # anchors, links, block attributes + ord('['): "{startsb}", + ord(']'): "{endsb}", + # superscript, subscript + ord('^'): "{caret}", + ord('~'): "{tilde}", + # bold + ord('*'): "{asterisk}", + # backslash + ord('\\'): "{backslash}", + # inline code + ord('`'): "{backtick}", +} +def asciidoc_escape(s: str) -> str: + s = s.translate(_asciidoc_escapes) + # :: is deflist item, ;; is has a replacement but no idea why + return s.replace("::", "{two-colons}").replace(";;", "{two-semicolons}") + +@dataclass(kw_only=True) +class List: + head: str + +@dataclass() +class Par: + sep: str + block_delim: str + continuing: bool = False + +class AsciiDocRenderer(Renderer): + __output__ = "asciidoc" + + _parstack: list[Par] + _list_stack: list[List] + _attrspans: list[str] + + def __init__(self, manpage_urls: Mapping[str, str], parser: Optional[markdown_it.MarkdownIt] = None): + super().__init__(manpage_urls, parser) + self._parstack = [ Par("\n\n", "====") ] + self._list_stack = [] + self._attrspans = [] + + def _enter_block(self, is_list: bool) -> None: + self._parstack.append(Par("\n+\n" if is_list else "\n\n", self._parstack[-1].block_delim + "=")) + def _leave_block(self) -> None: + self._parstack.pop() + def _break(self, force: bool = False) -> str: + result = self._parstack[-1].sep if force or self._parstack[-1].continuing else "" + self._parstack[-1].continuing = True + return result + + def _admonition_open(self, kind: str) -> str: + pbreak = self._break() + self._enter_block(False) + return f"{pbreak}[{kind}]\n{self._parstack[-2].block_delim}\n" + def _admonition_close(self) -> str: + self._leave_block() + return f"\n{self._parstack[-1].block_delim}\n" + + def _list_open(self, token: Token, head: str) -> str: + attrs = [] + if (idx := token.attrs.get('start')) is not None: + attrs.append(f"start={idx}") + if token.meta['compact']: + attrs.append('options="compact"') + if self._list_stack: + head *= len(self._list_stack[0].head) + 1 + self._list_stack.append(List(head=head)) + return f"{self._break()}[{','.join(attrs)}]" + def _list_close(self) -> str: + self._list_stack.pop() + return "" + + def text(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._parstack[-1].continuing = True + return asciidoc_escape(token.content) + def paragraph_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._break() + def paragraph_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return "" + def hardbreak(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return " +\n" + def softbreak(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return f" " + def code_inline(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._parstack[-1].continuing = True + return f"``{asciidoc_escape(token.content)}``" + def code_block(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self.fence(token, tokens, i, options, env) + def link_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._parstack[-1].continuing = True + return f"link:{quote(cast(str, token.attrs['href']), safe='/:')}[" + def link_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return "]" + def list_item_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._enter_block(True) + # allow the next token to be a block or an inline. + return f'\n{self._list_stack[-1].head} {{empty}}' + def list_item_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._leave_block() + return "\n" + def bullet_list_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._list_open(token, '*') + def bullet_list_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._list_close() + def em_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return "__" + def em_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return "__" + def strong_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return "**" + def strong_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return "**" + def fence(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + attrs = f"[source,{token.info}]\n" if token.info else "" + code = token.content + if code.endswith('\n'): + code = code[:-1] + return f"{self._break(True)}{attrs}----\n{code}\n----" + def blockquote_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + pbreak = self._break(True) + self._enter_block(False) + return f"{pbreak}[quote]\n{self._parstack[-2].block_delim}\n" + def blockquote_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._leave_block() + return f"\n{self._parstack[-1].block_delim}" + def note_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._admonition_open("NOTE") + def note_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._admonition_close() + def caution_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._admonition_open("CAUTION") + def caution_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._admonition_close() + def important_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._admonition_open("IMPORTANT") + def important_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._admonition_close() + def tip_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._admonition_open("TIP") + def tip_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._admonition_close() + def warning_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._admonition_open("WARNING") + def warning_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._admonition_close() + def dl_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return f"{self._break()}[]" + def dl_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return "" + def dt_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._break() + def dt_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._enter_block(True) + return ":: {empty}" + def dd_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return "" + def dd_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._leave_block() + return "\n" + def myst_role(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._parstack[-1].continuing = True + content = asciidoc_escape(token.content) + if token.meta['name'] == 'manpage' and (url := self._manpage_urls.get(token.content)): + return f"link:{quote(url, safe='/:')}[{content}]" + return f"[.{token.meta['name']}]``{asciidoc_escape(token.content)}``" + def inline_anchor(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._parstack[-1].continuing = True + return f"[[{token.attrs['id']}]]" + def attr_span_begin(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + self._parstack[-1].continuing = True + (id_part, class_part) = ("", "") + if id := token.attrs.get('id'): + id_part = f"[[{id}]]" + if s := token.attrs.get('class'): + if s == 'keycap': + class_part = "kbd:[" + self._attrspans.append("]") + else: + return super().attr_span_begin(token, tokens, i, options, env) + else: + self._attrspans.append("") + return id_part + class_part + def attr_span_end(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._attrspans.pop() + def heading_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return token.markup.replace("#", "=") + " " + def heading_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return "\n" + def ordered_list_open(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._list_open(token, '.') + def ordered_list_close(self, token: Token, tokens: Sequence[Token], i: int, options: OptionsDict, + env: MutableMapping[str, Any]) -> str: + return self._list_close() diff --git a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/options.py b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/options.py index d8a24b885f8730..f29d8fdb896825 100644 --- a/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/options.py +++ b/pkgs/tools/nix/nixos-render-docs/src/nixos_render_docs/options.py @@ -8,11 +8,13 @@ from markdown_it.utils import OptionsDict from markdown_it.token import Token from typing import Any, Optional +from urllib.parse import quote from xml.sax.saxutils import escape, quoteattr import markdown_it from . import parallel +from .asciidoc import AsciiDocRenderer, asciidoc_escape from .commonmark import CommonMarkRenderer from .docbook import DocBookRenderer, make_xml_id from .manpage import ManpageRenderer, man_escape @@ -476,6 +478,59 @@ def finalize(self) -> str: return "\n".join(result) +class OptionsAsciiDocRenderer(OptionDocsRestrictions, AsciiDocRenderer): + pass + +class AsciiDocConverter(BaseConverter): + __renderer__ = AsciiDocRenderer + __option_block_separator__ = "" + + def _parallel_render_prepare(self) -> Any: + return (self._manpage_urls, self._revision, self._markdown_by_default) + @classmethod + def _parallel_render_init_worker(cls, a: Any) -> AsciiDocConverter: + return cls(*a) + + def _render_code(self, option: dict[str, Any], key: str) -> list[str]: + # NOTE this duplicates the old direct-paste behavior, even if it is somewhat + # incorrect, since users rely on it. + if lit := option_is(option, key, 'literalDocBook'): + return [ f"*{key.capitalize()}:* {lit['text']}" ] + else: + return super()._render_code(option, key) + + def _render_description(self, desc: str | dict[str, Any]) -> list[str]: + # NOTE this duplicates the old direct-paste behavior, even if it is somewhat + # incorrect, since users rely on it. + if isinstance(desc, str) and not self._markdown_by_default: + return [ desc ] + else: + return super()._render_description(desc) + + def _related_packages_header(self) -> list[str]: + return [ "__Related packages:__" ] + + def _decl_def_header(self, header: str) -> list[str]: + return [ f"__{header}:__\n" ] + + def _decl_def_entry(self, href: Optional[str], name: str) -> list[str]: + if href is not None: + return [ f"* link:{quote(href, safe='/:')}[{asciidoc_escape(name)}]" ] + return [ f"* {asciidoc_escape(name)}" ] + + def _decl_def_footer(self) -> list[str]: + return [] + + def finalize(self) -> str: + result = [] + + for (name, opt) in self._sorted_options(): + result.append(f"== {asciidoc_escape(name)}\n") + result += opt.lines + result.append("\n\n") + + return "\n".join(result) + def _build_cli_db(p: argparse.ArgumentParser) -> None: p.add_argument('--manpage-urls', required=True) p.add_argument('--revision', required=True) @@ -498,6 +553,13 @@ def _build_cli_commonmark(p: argparse.ArgumentParser) -> None: p.add_argument("infile") p.add_argument("outfile") +def _build_cli_asciidoc(p: argparse.ArgumentParser) -> None: + p.add_argument('--manpage-urls', required=True) + p.add_argument('--revision', required=True) + p.add_argument('--markdown-by-default', default=False, action='store_true') + p.add_argument("infile") + p.add_argument("outfile") + def _run_cli_db(args: argparse.Namespace) -> None: with open(args.manpage_urls, 'r') as manpage_urls: md = DocBookConverter( @@ -537,11 +599,24 @@ def _run_cli_commonmark(args: argparse.Namespace) -> None: with open(args.outfile, 'w') as f: f.write(md.finalize()) +def _run_cli_asciidoc(args: argparse.Namespace) -> None: + with open(args.manpage_urls, 'r') as manpage_urls: + md = AsciiDocConverter( + json.load(manpage_urls), + revision = args.revision, + markdown_by_default = args.markdown_by_default) + + with open(args.infile, 'r') as f: + md.add_options(json.load(f)) + with open(args.outfile, 'w') as f: + f.write(md.finalize()) + def build_cli(p: argparse.ArgumentParser) -> None: formats = p.add_subparsers(dest='format', required=True) _build_cli_db(formats.add_parser('docbook')) _build_cli_manpage(formats.add_parser('manpage')) _build_cli_commonmark(formats.add_parser('commonmark')) + _build_cli_asciidoc(formats.add_parser('asciidoc')) def run_cli(args: argparse.Namespace) -> None: if args.format == 'docbook': @@ -550,5 +625,7 @@ def run_cli(args: argparse.Namespace) -> None: _run_cli_manpage(args) elif args.format == 'commonmark': _run_cli_commonmark(args) + elif args.format == 'asciidoc': + _run_cli_asciidoc(args) else: raise RuntimeError('format not hooked up', args) diff --git a/pkgs/tools/nix/nixos-render-docs/src/tests/test_asciidoc.py b/pkgs/tools/nix/nixos-render-docs/src/tests/test_asciidoc.py new file mode 100644 index 00000000000000..487506469954b2 --- /dev/null +++ b/pkgs/tools/nix/nixos-render-docs/src/tests/test_asciidoc.py @@ -0,0 +1,143 @@ +import nixos_render_docs + +from sample_md import sample1 + +class Converter(nixos_render_docs.md.Converter): + __renderer__ = nixos_render_docs.asciidoc.AsciiDocRenderer + +def test_lists() -> None: + c = Converter({}) + # attaching to the nth ancestor list requires n newlines before the + + assert c._render("""\ +- a + + b +- c + - d + - e + + 1 + + f +""") == """\ +[] +* {empty}a ++ +b + +* {empty}c ++ +[options="compact"] +** {empty}d ++ +[] +** {empty}e ++ +1 + + ++ +f +""" + +def test_full() -> None: + c = Converter({ 'man(1)': 'http://example.org' }) + assert c._render(sample1) == """\ +[WARNING] +==== +foo + +[NOTE] +===== +nested +===== + +==== + + +link:link[ multiline ] + +link:http://example.org[man(1)] reference + +[[b]]some [[a]]nested anchors + +__emph__ **strong** __nesting emph **and strong** and ``code``__ + +[] +* {empty}wide bullet + +* {empty}list + + +[] +. {empty}wide ordered + +. {empty}list + + +[options="compact"] +* {empty}narrow bullet + +* {empty}list + + +[options="compact"] +. {empty}narrow ordered + +. {empty}list + + +[quote] +==== +quotes + +[quote] +===== +with __nesting__ + +---- +nested code block +---- +===== + +[options="compact"] +* {empty}and lists + +* {empty} ++ +---- +containing code +---- + + +and more quote +==== + +[start=100,options="compact"] +. {empty}list starting at 100 + +. {empty}goes on + + +[] + +deflist:: {empty} ++ +[quote] +===== +with a quote and stuff +===== ++ +---- +code block +---- ++ +---- +fenced block +---- ++ +text + + +more stuff in same deflist:: {empty}foo +"""