support for module .mjs files

THIRD_PARTY_LICENSES.md

@@ -4297,7 +4297,7 @@ Apache License
 The following npm packages may be included in this product:
 
  - @aws-sdk/client-cognito-identity@3.600.0
- - @aws-sdk/client-sagemaker@3.600.0
+ - @aws-sdk/client-sagemaker@3.602.0
  - @aws-sdk/client-sso-oidc@3.600.0
  - @aws-sdk/client-sso@3.598.0
  - @aws-sdk/client-sts@3.600.0
@@ -5013,8 +5013,8 @@ The following npm packages may be included in this product:
  - @types/nlcst@2.0.3
  - @types/node-fetch@2.6.11
  - @types/node@17.0.45
- - @types/node@18.19.37
- - @types/node@20.14.6
+ - @types/node@18.19.38
+ - @types/node@20.14.7
  - @types/papaparse@5.3.14
  - @types/pg@8.11.6
  - @types/qs@6.9.15
@@ -5054,7 +5054,7 @@ MIT License
 
 The following npm package may be included in this product:
 
- - genaiscript-vscode@1.40.4
+ - genaiscript-vscode@1.41.7
 
 This package contains the following license and notice below:
 
@@ -18595,11 +18595,12 @@ third-party archives.
 
 -----------
 
-The following npm package may be included in this product:
+The following npm packages may be included in this product:
 
  - typescript@5.4.5
+ - typescript@5.5.2
 
-This package contains the following license and notice below:
+These packages each contain the following license and notice below:
 
 Apache License
 
@@ -22857,7 +22858,7 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 
 The following npm package may be included in this product:
 
- - electron-to-chromium@1.4.807
+ - electron-to-chromium@1.4.808
 
 This package contains the following license and notice below:
 
@@ -25157,11 +25158,11 @@ The following npm packages may be included in this product:
  - de-indent@1.0.2
  - degenerator@5.0.1
  - dlv@1.1.3
- - docs@1.40.4
+ - docs@1.41.7
  - eastasianwidth@0.2.0
- - genaiscript-core@1.40.4
- - genaiscript-sample@1.40.4
- - genaiscript@1.40.4
+ - genaiscript-core@1.41.7
+ - genaiscript-sample@1.41.7
+ - genaiscript@1.41.7
  - hast-util-is-body-ok-link@3.0.0
  - hast-util-to-string@3.0.0
  - html-whitespace-sensitive-tag-names@3.0.0
@@ -28304,7 +28305,7 @@ The following npm packages may be included in this product:
  - @volar/source-map@2.3.0
  - @volar/typescript@2.2.5
  - @vue/language-core@2.0.21
- - typescript-auto-import-cache@0.3.2
+ - typescript-auto-import-cache@0.3.3
 
 These packages each contain the following license and notice below:
 
@@ -28736,7 +28737,7 @@ SOFTWARE.
 
 The following npm package may be included in this product:
 
- - @astrojs/starlight@0.24.3
+ - @astrojs/starlight@0.24.4
 
 This package contains the following license and notice below:
 
@@ -29240,6 +29241,67 @@ SOFTWARE.
 
 -----------
 
+The following npm package may be included in this product:
+
+ - tsx@4.15.7
+
+This package contains the following license and notice below:
+
+MIT License
+
+Copyright (c) Hiroki Osame <hiroki.osame@gmail.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+-----------
+
+The following npm packages may be included in this product:
+
+ - get-tsconfig@4.7.5
+ - resolve-pkg-maps@1.0.0
+
+These packages each contain the following license and notice below:
+
+MIT License
+
+Copyright (c) Hiroki Osame <hiroki.osame@gmail.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+-----------
+
 The following npm packages may be included in this product:
 
  - hasown@2.0.2
@@ -31992,7 +32054,7 @@ THE SOFTWARE.
 
 The following npm package may be included in this product:
 
- - is-core-module@2.13.1
+ - is-core-module@2.14.0
 
 This package contains the following license and notice below:
 
@@ -34141,7 +34203,7 @@ THE SOFTWARE.
 
 The following npm package may be included in this product:
 
- - vue-router@4.3.3
+ - vue-router@4.4.0
 
 This package contains the following license and notice below:
 

packages/cli/package.json

@@ -28,6 +28,7 @@
     "pdfjs-dist": "4.3.136",
     "promptfoo": "^0.65.2",
     "tree-sitter-wasms": "^0.1.11",
+    "tsx": "^4.15.7",
     "typescript": "5.4.5",
     "web-tree-sitter": "^0.22.2",
     "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.2/xlsx-0.20.2.tgz"
@@ -68,11 +69,10 @@
     "prompts": "^2.4.2",
     "replace-ext": "^2.0.0",
     "semver": "^7.6.2",
-    "tsx": "^4.15.6",
     "zx": "^8.1.2"
   },
   "scripts": {
-    "compile": "esbuild src/main.ts --bundle --platform=node --target=node20 --outfile=built/genaiscript.cjs --external:dockerode --external:pdfjs-dist --external:llamaindex --external:web-tree-sitter --external:tree-sitter-wasms --external:promptfoo && node ../../scripts/patch-cli.mjs",
+    "compile": "esbuild src/main.ts --bundle --platform=node --target=node20 --outfile=built/genaiscript.cjs --external:tsx --external:dockerode --external:pdfjs-dist --external:llamaindex --external:web-tree-sitter --external:tree-sitter-wasms --external:promptfoo && node ../../scripts/patch-cli.mjs",
     "postcompile": "node built/genaiscript.cjs info help > ../../docs/src/content/docs/reference/cli/commands.md",
     "go": "yarn compile && node built/genaiscript.cjs",
     "test": "node --import tsx --test src/**.test.ts",

packages/core/package.json

@@ -36,6 +36,7 @@
     "gpt-tokenizer": "^2.1.2",
     "html-escaper": "^3.0.3",
     "html-to-text": "^9.0.5",
+    "ignore": "^5.3.1",
     "inflection": "^3.0.0",
     "ini": "^4.1.3",
     "json5": "^2.2.3",
@@ -55,10 +56,10 @@
     "toml": "^3.0.0",
     "tree-sitter-wasms": "^0.1.11",
     "ts-dedent": "^2.2.0",
+    "tsx": "^4.15.7",
     "typescript": "5.4.5",
     "xlsx": "^0.18.5",
-    "yaml": "^2.4.5",
-    "ignore": "^5.3.1"
+    "yaml": "^2.4.5"
   },
   "scripts": {
     "typecheck": "tsc -p src",

packages/core/src/aici.ts

@@ -1,4 +1,5 @@
 import {
+    ChatCompletionContentPartText,
     ChatCompletionHandler,
     ChatCompletionResponse,
     LanguageModel,
@@ -9,7 +10,6 @@ import { fromHex, logError, normalizeInt, utf8Decode } from "./util"
 import { AICI_CONTROLLER, TOOL_ID } from "./constants"
 import { LanguageModelConfiguration, host } from "./host"
 import { NotSupportedError, RequestError } from "./error"
-import { ChatCompletionContentPartText } from "openai/resources"
 import { createFetch } from "./fetch"
 import { parseModelIdentifier } from "./models"
 

packages/core/src/chat.ts

@@ -19,6 +19,10 @@ import { estimateChatTokens } from "./tokens"
 import { createChatTurnGenerationContext } from "./runpromptcontext"
 import { dedent } from "./indent"
 
+export type ChatCompletionContentPartText = OpenAI.Chat.Completions.ChatCompletionContentPartText
+
+export type ChatCompletionContentPart = OpenAI.Chat.Completions.ChatCompletionContentPart
+
 export type ChatCompletionTool = OpenAI.Chat.Completions.ChatCompletionTool
 
 export type ChatCompletionChunk = OpenAI.Chat.Completions.ChatCompletionChunk

packages/core/src/constants.ts

@@ -17,6 +17,7 @@ export const GENAI_JS_EXT = ".genai.js"
 export const GENAI_MJS_EXT = ".genai.mjs"
 export const GENAI_JS_GLOB = "**/*.genai.*js"
 export const GENAI_JS_REGEX = /\.genai\.m?js$/i
+export const MODULE_JS_REGEX = /\.mjs$/i
 export const GPSPEC_REGEX = /\.gpspec\.md$/i
 export const HTTPS_REGEX = /^https:\/\//i
 export const CSV_REGEX = /\.(t|c)sv$/i

packages/core/src/expander.ts

@@ -3,7 +3,12 @@ import { assert, normalizeFloat, normalizeInt, unique } from "./util"
 import { MarkdownTrace } from "./trace"
 import { errorMessage, isCancelError } from "./error"
 import { estimateTokens } from "./tokens"
-import { MAX_TOOL_CALLS, MODEL_PROVIDER_AICI, SYSTEM_FENCE } from "./constants"
+import {
+    MAX_TOOL_CALLS,
+    MODEL_PROVIDER_AICI,
+    MODULE_JS_REGEX,
+    SYSTEM_FENCE,
+} from "./constants"
 import { PromptImage, renderPromptNode } from "./promptdom"
 import { GenerationOptions, createPromptContext } from "./promptcontext"
 import { evalPrompt } from "./evalprompt"
@@ -116,11 +121,9 @@ async function callExpander(
     }
 
     try {
-        if (/^export\s+default\s+/m.test(r.jsSource)) {
-            if (!/\.mjs$/i.test(r.filename))
-                throw new Error("export default requires .mjs file")
+        if (MODULE_JS_REGEX.test(r.filename))
             await importPrompt(ctx, r, { logCb })
-        } else {
+        else {
             await evalPrompt(ctx, r, {
                 sourceMaps: true,
                 logCb,

packages/core/src/importprompt.ts

@@ -1,6 +1,7 @@
 import { assert } from "console"
 import { host } from "./host"
 import { logError } from "./util"
+import { tsImport } from "tsx/esm/api"
 
 function resolveGlobal(): any {
     if (typeof window !== "undefined")
@@ -25,20 +26,20 @@ export async function importPrompt(
     try {
         // override global context
         for (const field of Object.keys(ctx0)) {
-            assert(!glb[field])
+            assert(field === "console" || !glb[field], `overriding global field ${field}`)
             oldGlb[field] = glb[field]
             glb[field] = (ctx0 as any)[field]
         }
 
         const modulePath = filename.startsWith("/")
             ? filename
             : host.path.join(host.projectFolder(), filename)
-        const module = await import(modulePath)
+        const parentURL = import.meta.url || new URL(__filename, "file://").href
+        const module = await tsImport(modulePath, {
+            parentURL,
+        })
         const main = module.default
-        if (!main) throw new Error("default import function missing")
-        if (typeof main !== "function")
-            throw new Error("default export must be a function")
-        await main(ctx0)
+        if (typeof main === "function") await main(ctx0)
     } catch (err) {
         logError(err)
         throw err

packages/core/src/tokens.ts

@@ -1,12 +1,11 @@
-import { ChatCompletionMessageParam } from "./chat"
 import {
     ChatCompletionContentPart,
     ChatCompletionContentPartText,
-    type ChatCompletionTool,
-} from "openai/resources"
+    ChatCompletionMessageParam,
+    ChatCompletionTool,
+} from "./chat"
 import { encodeChat, encode } from "gpt-tokenizer"
-import { logError, logVerbose } from "./util"
-import { parseModelIdentifier } from "./models"
+import { logVerbose } from "./util"
 
 export function estimateTokens(model: string, text: string) {
     if (!text?.length) return 0

packages/core/src/util.ts

@@ -56,7 +56,7 @@ export function assert(
     debugData?: any
 ) {
     if (!cond) {
-        if (debugData) console.error(`assertion failed ${msg}`, debugData)
+        if (debugData) console.error(msg || `assertion failed`, debugData)
         // eslint-disable-next-line no-debugger
         debugger
         throw new Error(msg)

packages/sample/genaisrc/mjs.genai.mjs

@@ -0,0 +1,13 @@
+script({
+    title: "top-level-mjs",
+    model: "openai:gpt-3.5-turbo",
+    files: ["src/rag/markdown.md"],
+    tests: {
+        files: ["src/rag/markdown.md"],
+        keywords: "markdown",
+    },
+})
+
+import { summarize } from "./summarizer.mjs"
+
+summarize(env.files)

packages/sample/genaisrc/summarizer.mjs

@@ -0,0 +1,4 @@
+export function summarize(files) {
+    def("FILE", files)
+    $`Summarize each file. Be concise.`
+}

packages/tsconfig-base.json

@@ -1,6 +1,6 @@
 {
     "compilerOptions": {
-        "moduleResolution": "node",
+        "moduleResolution": "bundler",
         "target": "ES2022",
         "module": "ES2022",
         "lib": [

packages/vscode/package.json

@@ -357,7 +357,7 @@
         "vscode:update-dts": "npx @vscode/dts dev && mv vscode.*.d.ts src/",
         "vscode:prepublish": "yarn run compile",
         "compile:icons": "node updatefonts.mjs",
-        "compile:extension": "esbuild src/extension.ts --sourcemap --bundle --format=cjs --platform=node --target=node20 --outfile=built/extension.js --external:vscode --external:pdfjs-dist --external:llamaindex --external:web-tree-sitter --external:tree-sitter-wasms",
+        "compile:extension": "esbuild src/extension.ts --sourcemap --bundle --format=cjs --platform=node --target=node20 --outfile=built/extension.js --external:tsx --external:vscode --external:pdfjs-dist --external:llamaindex --external:web-tree-sitter --external:tree-sitter-wasms",
         "compile": "yarn compile:icons && yarn compile:extension",
         "lint": "eslint src --ext ts",
         "run-in-browser": "vscode-test-web --browserType=chromium --extensionDevelopmentPath=. ../sample",
@@ -391,6 +391,7 @@
         "mammoth": "^1.8.0",
         "pdfjs-dist": "4.3.136",
         "tree-sitter-wasms": "^0.1.11",
+        "tsx": "^4.15.7",
         "web-tree-sitter": "^0.22.6"
     },
     "optionalDependencies": {