📑 Add literalinclude directive (#610)

Supports `literalinclude` directive and options across RST and Sphinx.

- [RST documentation](https://docutils.sourceforge.io/docs/ref/rst/directives.html#including-an-external-document-fragment)
- [Sphinx documentation](https://www.sphinx-doc.org/en/master/usage/restructuredtext/directives.html#directive-literalinclude)

See executablebooks/jupyterlab-myst#189 and https://github.com/orgs/executablebooks/discussions/1026

.changeset/cool-cooks-act.md

@@ -0,0 +1,6 @@
+---
+'myst-directives': patch
+'myst-transforms': patch
+---
+
+Move includeDirective transform to myst-transforms and make it generic for use in JupyterLab

.changeset/flat-suits-allow.md

@@ -0,0 +1,5 @@
+---
+'myst-spec-ext': patch
+---
+
+Add `include` node, that implements the `literalinclude` directive

.changeset/gold-weeks-appear.md

@@ -0,0 +1,5 @@
+---
+'myst-directives': patch
+---
+
+Remove the codeBlockDirective, this is now the same as the `codeDirective`.

.changeset/orange-planes-promise.md

@@ -0,0 +1,6 @@
+---
+'myst-directives': patch
+'myst-cli': patch
+---
+
+Add `literalinclude` directive

docs/code.md

@@ -83,13 +83,79 @@ caption (string)
 name (string)
 : The target label for the code-block, can be used by `ref` and `numref` roles.
 
-```{note}
+```{note} Alternative implementations
 :class: dropdown
-# Alternative implementations
 
 The parser also supports the `docutils` implementation (see [docutils documentation](https://docutils.sourceforge.io/docs/ref/rst/directives.html#code)) of a `{code}` directive, which only supports the `number-lines` option.
 
 It is recommended to use the more fully featured `code-block` directive documented above, or a simple markdown code block.
 
 All implementations are resolved to the same `code` type in the abstract syntax tree.
 ```
+
+## Including Files
+
+If your code is in a separate file you can use the `literalinclude` directive (or the `include` directive with the `literal` flag).
+This directive is helpful for showing code snippets without duplicating your content.
+
+For example, a `literalinclude` of a snippet of the `myst.yml` such as:
+
+````markdown
+```{literalinclude} myst.yml
+:start-at: project
+:end-before: references
+:lineno-match:
+```
+````
+
+creates a snippet that has matching line numbers, and starts at a line including `"project"` and ends before the line including `"references"`.
+
+```{literalinclude} myst.yml
+:start-at: project
+:end-before: references
+:lineno-match:
+```
+
+:::{note} Auto Reload
+If you are working with the auto-reload (e.g. `myst start`), currently you will need to save the file with the `literalinclude` directive for the contents to update.code for the contents to update.
+:::
+
+## `include` Reference
+
+The argument of an include directive is the file path, relative to the file from which it was referenced.
+By default the file will be parsed using MyST, you can also set the file to be `literal`, which will show as a code-block; this is the same as using the `literalinclude` directive.
+If in literal mode, the directive also accepts all of the options from the `code-block` (e.g. `:linenos:`).
+
+To select a portion of the file to be shown using the `start-at`/`start-after` selectors with the `end-before`/`end-at`, which use a snippet of included text.
+Alternatively, you can explicitly select the lines (e.g. `1,3,5-10,20-`) or the `start-line`/`end-line` (which is zero based for compatibility with Sphinx).
+
+literal (boolean)
+: Flag the include block as literal, and show the contents as a code block. This can also be set automatically by setting the `language` or using the `literalinclude` directive.
+
+lang (string)
+: The language of the code to be highlighted as. If set, this automatically changes an `include` into a `literalinclude`.
+: You can alias this as `language` or `code`
+
+start-line (number)
+: Only the content starting from this line will be included. The first line has index 0 and negative values count from the end.
+
+start-at (string)
+: Only the content after and including the first occurrence of the specified text in the external data file will be included.
+
+start-after (string)
+: Only the content after the first occurrence of the specified text in the external data file will be included.
+
+end-line (number)
+: Only the content up to (but excluding) this line will be included.
+
+end-at (string)
+: Only the content up to and including the first occurrence of the specified text in the external data file (but after any start-after text) will be included.
+
+end-before (string)
+: Only the content before the first occurrence of the specified text in the external data file (but after any start-after text) will be included.
+
+lines (string)
+: Specify exactly which lines to include from the original file, starting at 1. For example, `1,3,5-10,20-` includes the lines 1, 3, 5 to 10 and lines 20 to the last line of the original file.
+
+lineno-match (boolean)
+: Display the original line numbers, correct only when the selection consists of contiguous lines.

packages/myst-cli/src/process/mdast.ts

@@ -37,7 +37,7 @@ import {
   checkLinksTransform,
   embedTransform,
   importMdastFromJson,
-  includeFilesDirective,
+  includeFilesTransform,
   liftCodeMetadataToBlock,
   transformLinkedDOIs,
   transformOutputs,
@@ -155,7 +155,7 @@ export async function transformMdast(
   cache.$internalReferences[file] = state;
   // Import additional content from mdast or other files
   importMdastFromJson(session, file, mdast);
-  includeFilesDirective(session, file, mdast);
+  includeFilesTransform(session, file, mdast, vfile);
   // This needs to come before basic transformations since it may add labels to blocks
   liftCodeMetadataToBlock(session, file, mdast);
 

packages/myst-cli/src/transforms/include.ts

@@ -1,27 +1,29 @@
+import path from 'node:path';
 import fs from 'node:fs';
-import type { GenericNode, GenericParent } from 'myst-common';
+import type { GenericParent } from 'myst-common';
+import { fileError } from 'myst-common';
 import { parseMyst } from '../process/index.js';
-import { selectAll } from 'unist-util-select';
-import { join, dirname } from 'node:path';
 import type { ISession } from '../session/types.js';
+import type { VFile } from 'vfile';
+import { includeDirectiveTransform } from 'myst-transforms';
 
-/**
- * This is the {include} directive, that loads from disk.
- *
- * RST documentation:
- *  - https://docutils.sourceforge.io/docs/ref/rst/directives.html#including-an-external-document-fragment
- */
-export function includeFilesDirective(session: ISession, filename: string, mdast: GenericParent) {
-  const includeNodes = selectAll('include', mdast) as GenericNode[];
-  const dir = dirname(filename);
-  includeNodes.forEach((node) => {
-    const file = join(dir, node.file);
-    if (!fs.existsSync(file)) {
-      session.log.error(`Include Directive: Could not find "${file}" in "${filename}"`);
+export function includeFilesTransform(
+  session: ISession,
+  baseFile: string,
+  tree: GenericParent,
+  vfile: VFile,
+) {
+  const dir = path.dirname(baseFile);
+  const loadFile = (filename: string) => {
+    const fullFile = path.join(dir, filename);
+    if (!fs.existsSync(fullFile)) {
+      fileError(vfile, `Include Directive: Could not find "${fullFile}" in "${baseFile}"`);
       return;
     }
-    const content = fs.readFileSync(file).toString();
-    const children = parseMyst(session, content, filename).children as GenericNode[];
-    node.children = children;
-  });
+    return fs.readFileSync(fullFile).toString();
+  };
+  const parseContent = (filename: string, content: string) => {
+    return parseMyst(session, content, filename).children;
+  };
+  includeDirectiveTransform(tree, vfile, { loadFile, parseContent });
 }

packages/myst-directives/package.json

@@ -23,6 +23,8 @@
     "clean": "rimraf dist",
     "lint": "eslint \"src/**/!(*.spec).ts\" -c ./.eslintrc.cjs",
     "lint:format": "npx prettier --check \"src/**/*.ts\"",
+    "test": "vitest run",
+    "test:watch": "vitest watch",
     "build:esm": "tsc --project ./tsconfig.json --module es2015 --outDir dist --declaration",
     "build": "npm-run-all -l clean -p build:esm"
   },

packages/myst-directives/src/code.spec.ts

@@ -0,0 +1,54 @@
+import { describe, expect, test } from 'vitest';
+import { getCodeBlockOptions } from './code.js';
+import { VFile } from 'vfile';
+
+describe('Code block options', () => {
+  test('default options', () => {
+    const vfile = new VFile();
+    const opts = getCodeBlockOptions({}, vfile);
+    expect(opts).toEqual({});
+    expect(vfile.messages.length).toEqual(0);
+  });
+  test('number-lines', () => {
+    const vfile = new VFile();
+    const opts = getCodeBlockOptions({ 'number-lines': 1 }, vfile);
+    expect(opts).toEqual({ showLineNumbers: true });
+    expect(vfile.messages.length).toEqual(0);
+  });
+  test('number-lines: 2', () => {
+    const vfile = new VFile();
+    const opts = getCodeBlockOptions({ 'number-lines': 2 }, vfile);
+    expect(opts).toEqual({ showLineNumbers: true, startingLineNumber: 2 });
+    expect(vfile.messages.length).toEqual(0);
+  });
+  test('number-lines clashes with lineno-start', () => {
+    const vfile = new VFile();
+    const opts = getCodeBlockOptions({ 'number-lines': 1, 'lineno-start': 2 }, vfile);
+    expect(opts).toEqual({ showLineNumbers: true, startingLineNumber: 2 });
+    // Show warning!
+    expect(vfile.messages.length).toEqual(1);
+  });
+  test('lineno-start activates showLineNumbers', () => {
+    const vfile = new VFile();
+    const opts = getCodeBlockOptions({ 'lineno-start': 1 }, vfile);
+    expect(opts).toEqual({ showLineNumbers: true });
+    expect(vfile.messages.length).toEqual(0);
+  });
+  test('emphasize-lines', () => {
+    const vfile = new VFile();
+    const opts = getCodeBlockOptions({ 'emphasize-lines': '3,5' }, vfile);
+    expect(opts).toEqual({ emphasizeLines: [3, 5] });
+    expect(vfile.messages.length).toEqual(0);
+  });
+  // See https://github.com/executablebooks/jupyterlab-myst/issues/174
+  test(':lineno-start: 10, :emphasize-lines: 12,13', () => {
+    const vfile = new VFile();
+    const opts = getCodeBlockOptions({ 'lineno-start': 10, 'emphasize-lines': '12,13' }, vfile);
+    expect(opts).toEqual({
+      showLineNumbers: true,
+      emphasizeLines: [12, 13],
+      startingLineNumber: 10,
+    });
+    expect(vfile.messages.length).toEqual(0);
+  });
+});