feature: engine extensions and quarto api

dev-docs/subtree-extensions.md

@@ -0,0 +1,41 @@
+# Subtree extensions
+
+Subtree extensions live in `src/resources/subtree-extensions`.
+
+Each is the complete git repository of a Quarto extension, e.g. the directory tree for the Julia engine looks like
+
+```
+src/resources/extension-subtrees/
+  julia-engine/
+    _extensions/
+      julia-engine/
+        _extension.yml
+        julia-engine.ts
+        ...
+```
+
+The command to add or update a subtree is
+
+```
+quarto dev-call pull-git-subtree subtree-name
+```
+
+Omit _subtree-name_ to add/update all.
+
+The code in `src/command/dev-call/pull-git-subtree/cmd.ts` contains a table of subtree
+
+- `name`
+- `prefix` (subdirectory)
+- `remoteUrl`
+- `remoteBranch`
+
+If the command is successful, it will add two commits, one the squashed changes from the remote repo and one a merge commit.
+
+The commits have subtree status information in the message and metadata, so don't change them.
+
+The commits can't be rebased -- you'll get weird errors indicating it tried to merge changes at the root of the repo.
+
+So you must either
+
+- run the command when ready to merge to main, or
+- remove the commits when rebasing, and run the `dev-call` command again

package/src/common/prepare-dist.ts

@@ -118,6 +118,11 @@ export async function prepareDist(
   supportingFiles(config);
   info("");
 
+  // Update extension-build import map for distribution
+  info("Updating extension-build import map");
+  updateImportMap(config);
+  info("");
+
   // Create the deno bundle
   // const input = join(config.directoryInfo.src, "quarto.ts");
   const output = join(config.directoryInfo.pkgWorking.bin, "quarto.js");
@@ -163,6 +168,20 @@ export async function prepareDist(
     );
   }
 
+  // Copy quarto-types to extension-build directory
+  // Note: deno.json and import-map.json are copied by supportingFiles() and
+  // import-map.json is then updated by updateImportMap() for distribution
+  info("Copying quarto-types.d.ts to extension-build directory");
+  const extensionBuildDir = join(
+    config.directoryInfo.pkgWorking.share,
+    "extension-build",
+  );
+  copySync(
+    join(config.directoryInfo.root, "packages/quarto-types/dist/index.d.ts"),
+    join(extensionBuildDir, "quarto-types.d.ts"),
+    { overwrite: true },
+  );
+
   // Remove the config directory, if present
   info(`Cleaning config`);
   const configDir = join(config.directoryInfo.dist, "config");
@@ -209,6 +228,49 @@ function supportingFiles(config: Configuration) {
   pathsToClean.forEach((path) => Deno.removeSync(path, { recursive: true }));
 }
 
+function updateImportMap(config: Configuration) {
+  // Read the import map that was copied from src/resources/extension-build/
+  const importMapPath = join(
+    config.directoryInfo.pkgWorking.share,
+    "extension-build",
+    "import-map.json",
+  );
+  const importMapContent = JSON.parse(Deno.readTextFileSync(importMapPath));
+
+  // Read the source import map to get current Deno std versions
+  const sourceImportMapPath = join(config.directoryInfo.src, "import_map.json");
+  const sourceImportMap = JSON.parse(Deno.readTextFileSync(sourceImportMapPath));
+  const sourceImports = sourceImportMap.imports as Record<string, string>;
+
+  // Update the import map for distribution:
+  // 1. Change @quarto/types path from dev (../../../packages/...) to dist (./quarto-types.d.ts)
+  // 2. Update all other imports (Deno std versions) from source import map
+  const updatedImports: Record<string, string> = {
+    "@quarto/types": "./quarto-types.d.ts",
+  };
+
+  // Copy all other imports from source, updating versions
+  for (const key in importMapContent.imports) {
+    if (key !== "@quarto/types") {
+      const sourceValue = sourceImports[key];
+      if (!sourceValue) {
+        throw new Error(
+          `Import map key "${key}" not found in source import_map.json`,
+        );
+      }
+      updatedImports[key] = sourceValue;
+    }
+  }
+
+  importMapContent.imports = updatedImports;
+
+  // Write back the updated import map
+  Deno.writeTextFileSync(
+    importMapPath,
+    JSON.stringify(importMapContent, null, 2) + "\n",
+  );
+}
+
 interface Filter {
   // The name of the filter (the LUA file and perhaps the directory)
   name: string;

package/src/quarto-bld

@@ -17,4 +17,5 @@ export DENO_NO_UPDATE_CHECK=1
 
 # TODO: Consider generating a source map or something to get a good stack
 # Create the Deno bundle
-"$QUARTO_DENO" run --no-check --unstable-kv --unstable-ffi --allow-all --v8-flags=--stack-trace-limit=100 --importmap="${SCRIPT_PATH}/../../src/import_map.json" "$SCRIPT_PATH/bld.ts" $@
+# --enable-experimental-regexp-engine is required for /regex/l, https://github.com/quarto-dev/quarto-cli/issues/9737
+"$QUARTO_DENO" run --no-check --unstable-kv --unstable-ffi --allow-all --v8-flags=--enable-experimental-regexp-engine,--stack-trace-limit=100 --importmap="${SCRIPT_PATH}/../../src/import_map.json" "$SCRIPT_PATH/bld.ts" $@

package/src/quarto-bld.cmd

@@ -9,4 +9,5 @@ if NOT DEFINED QUARTO_DENO (
 SET "RUST_BACKTRACE=full"
 SET "DENO_NO_UPDATE_CHECK=1"
 
-"%QUARTO_DENO%" run --unstable-kv --unstable-ffi --allow-all --importmap=%~dp0\..\..\src\import_map.json %~dp0\bld.ts %*
+REM --enable-experimental-regexp-engine is required for /regex/l, https://github.com/quarto-dev/quarto-cli/issues/9737
+"%QUARTO_DENO%" run --no-check --unstable-kv --unstable-ffi --allow-all --v8-flags=--enable-experimental-regexp-engine,--stack-trace-limit=100 --importmap=%~dp0\..\..\src\import_map.json %~dp0\bld.ts %*

package/src/util/deno.ts

@@ -54,6 +54,8 @@ export async function compile(
   denoBundleCmd.push("compile");
   denoBundleCmd.push("--unstable-kv");
   denoBundleCmd.push("--unstable-ffi");
+  // --enable-experimental-regexp-engine is required for /regex/l, https://github.com/quarto-dev/quarto-cli/issues/9737
+  denoBundleCmd.push("--v8-flags=--enable-experimental-regexp-engine,--stack-trace-limit=100");
   denoBundleCmd.push(
     "--importmap=" + configuration.importmap,
   );

packages/quarto-types/.gitignore

@@ -0,0 +1,20 @@
+# Dependency directories
+node_modules/
+npm-debug.log
+yarn-error.log
+yarn-debug.log
+
+# Output directories
+
+# TypeScript cache
+*.tsbuildinfo
+
+# IDE files
+.idea/
+.vscode/
+*.swp
+*.swo
+
+# OS specific files
+.DS_Store
+Thumbs.db

packages/quarto-types/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Quarto
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

packages/quarto-types/README.md

@@ -0,0 +1,100 @@
+# @quarto/types
+
+TypeScript type definitions for developing Quarto execution engines.
+
+## Installation
+
+The goal in a couple of releases is for you to be able to do
+
+```bash
+npm install @quarto/types
+```
+
+But this package is not published yet, because the interfaces are still in flux.
+
+Instead, build the package and copy dist/index.d.ts to types/quarto-types.d.ts in your engine repo.
+
+```bash
+npm run build
+```
+
+## Usage
+
+This package provides TypeScript type definitions for implementing custom execution engines for Quarto.
+
+```typescript
+import { ExecutionEngine, EngineProjectContext } from "@quarto/types";
+
+export const customEngine: ExecutionEngine = {
+  name: "custom",
+  defaultExt: ".qmd",
+
+  // Implement required methods...
+
+  target: (
+    file: string,
+    quiet: boolean | undefined,
+    markdown: MappedString | undefined,
+    project: EngineProjectContext // Using the restricted context interface
+  ) => {
+    // Implementation...
+  },
+};
+```
+
+## Core Interfaces
+
+### ExecutionEngineDiscovery
+
+Interface for implementing a Quarto execution engine discovery, responsible for determining which engine should handle a file and launching engine instances.
+
+### ExecutionEngineInstance
+
+Interface for a launched execution engine that can execute documents within a project context.
+
+### EngineProjectContext
+
+Restricted version of Quarto's ProjectContext that only exposes functionality needed by execution engines.
+
+## Type Simplifications
+
+This package is designed to be a standalone, lightweight type library for external engine development. To maintain this independence, some internal Quarto types are intentionally simplified:
+
+### Simplified types with omitted fields
+
+- **`RenderOptions`**: The internal Quarto version includes a `services: RenderServices` field, which provides access to temp file context, extension context, and notebook context. This field has been omitted as it requires pulling in large portions of Quarto's internal type system. All other fields are included. See [src/command/render/types.ts](https://github.com/quarto-dev/quarto-cli/blob/main/src/command/render/types.ts#L29-L42) for the full type.
+
+### Simplified to `Record<string, unknown>` or index signatures
+
+- **`Format.render`**: The full internal type is `FormatRender` with 100+ lines of specific rendering options including brand configuration and CSS handling. See [src/config/types.ts](https://github.com/quarto-dev/quarto-cli/blob/main/src/config/types.ts#L463-L525).
+
+- **`Format.execute`**: The full internal type is `FormatExecute` with specific execution options. See [src/config/types.ts](https://github.com/quarto-dev/quarto-cli/blob/main/src/config/types.ts#L527-L561).
+
+- **`Format.pandoc`**: The full internal type is `FormatPandoc` with 80+ lines of pandoc-specific options. Simplified to `{ to?: string; [key: string]: unknown }` preserving only the most commonly accessed `to` property. See [src/config/types.ts](https://github.com/quarto-dev/quarto-cli/blob/main/src/config/types.ts#L563-L646).
+
+When accessing properties from these records, use type assertions as needed:
+
+```typescript
+const figFormat = options.format.execute["fig-format"] as string | undefined;
+const keepHidden = options.format.render?.["keep-hidden"] as
+  | boolean
+  | undefined;
+const writer = options.format.pandoc.to; // Type-safe access to 'to'
+const standalone = options.format.pandoc["standalone"] as boolean | undefined;
+```
+
+### Omitted optional functions
+
+The full internal `Format` interface includes several optional functions that are used internally by Quarto but are not needed by external engines:
+
+- **`mergeAdditionalFormats`**: Internal format merging logic
+- **`resolveFormat`**: Format resolution and normalization
+- **`formatExtras`**: Compute format-specific extras (requires `RenderServices` and `ProjectContext`)
+- **`formatPreviewFile`**: Preview file name transformation
+- **`extensions`**: Format-specific extension configuration
+
+These functions require deep integration with Quarto's internal systems and are not exposed to external engines. See [src/config/types.ts](https://github.com/quarto-dev/quarto-cli/blob/main/src/config/types.ts#L420-L456) for the full `Format` interface.
+
+## License
+
+MIT