configurable default model

packages/cli/src/cli.ts

@@ -66,8 +66,8 @@ export async function cli() {
     }
 
     let nodeHost: NodeHost
-    program.hook("preAction", (cmd) => {
-        nodeHost = NodeHost.install(cmd.opts().env)
+    program.hook("preAction", async (cmd) => {
+        nodeHost = await NodeHost.install(cmd.opts().env)
     })
     program
         .name(TOOL_ID)

packages/cli/src/info.ts

@@ -1,6 +1,5 @@
 import {
     CORE_VERSION,
-    DEFAULT_MODEL,
     ModelConnectionInfo,
     YAMLStringify,
     host,
@@ -23,7 +22,7 @@ async function resolveScriptsConnectionInfo(
     const models: Record<string, ModelConnectionOptions> = {}
     for (const template of templates) {
         const conn: ModelConnectionOptions = {
-            model: template.model ?? DEFAULT_MODEL,
+            model: template.model ?? host.defaultModelOptions.model,
         }
         const key = JSON.stringify(conn)
         if (!models[key]) models[key] = conn

packages/cli/src/llamaindexretrieval.ts

@@ -29,6 +29,7 @@ import {
     PDF_MIME_TYPE,
     DOCX_MIME_TYPE,
     JSON_SCHEMA_MIME_TYPE,
+    assert,
 } from "genaiscript-core"
 import { type BaseReader } from "llamaindex"
 import type { GenericFileSystem } from "@llamaindex/env"
@@ -143,6 +144,7 @@ export class LlamaIndexRetrievalService
     }
 
     private async getModelToken(modelId: string) {
+        assert(!!modelId)
         const { provider } = parseModelIdentifier(modelId)
         const conn = await this.host.getLanguageModelConfiguration(modelId)
         if (provider === MODEL_PROVIDER_OLLAMA)
@@ -151,6 +153,7 @@ export class LlamaIndexRetrievalService
     }
 
     async pullModel(modelid: string): Promise<ResponseStatus> {
+        assert(!!modelid)
         const { provider, model } = parseModelIdentifier(modelid)
         const conn = await this.getModelToken(modelid)
         if (provider === MODEL_PROVIDER_OLLAMA) {

packages/cli/src/nodehost.ts

@@ -4,6 +4,8 @@ import {
     AZURE_OPENAI_TOKEN_SCOPES,
     AbortSignalOptions,
     AskUserOptions,
+    DEFAULT_MODEL,
+    DEFAULT_TEMPERATURE,
     Host,
     LanguageModel,
     LanguageModelConfiguration,
@@ -19,6 +21,7 @@ import {
     UTF8Encoder,
     createBundledParsers,
     createFileSystem,
+    parseDefaultsFromEnv,
     parseTokenFromEnv,
     resolveLanguageModel,
     setHost,
@@ -56,14 +59,18 @@ export class NodeHost implements Host {
     readonly workspace = createFileSystem()
     readonly parser = createBundledParsers()
     readonly docker = new DockerManager()
+    readonly defaultModelOptions = {
+        model: DEFAULT_MODEL,
+        temperature: DEFAULT_TEMPERATURE,
+    }
 
     constructor() {
         const srv = new LlamaIndexRetrievalService(this)
         this.retrieval = srv
         this.models = srv
     }
 
-    static install(dotEnvPath: string) {
+    static async install(dotEnvPath: string) {
         dotEnvPath = dotEnvPath || resolve(".env")
         if (existsSync(dotEnvPath)) {
             const res = dotenv.config({
@@ -75,19 +82,25 @@ export class NodeHost implements Host {
         }
         const h = new NodeHost()
         setHost(h)
+        await h.parseDefaults()
         return h
     }
 
     async readSecret(name: string): Promise<string | undefined> {
         return process.env[name]
     }
 
+    private async parseDefaults() {
+        await parseDefaultsFromEnv(process.env)
+    }
+
     private _azureToken: AccessToken
     async getLanguageModelConfiguration(
         modelId: string,
         options?: { token?: boolean } & AbortSignalOptions & TraceOptions
     ): Promise<LanguageModelConfiguration> {
         const { signal, token: askToken } = options || {}
+        await this.parseDefaults()
         const tok = await parseTokenFromEnv(process.env, modelId)
         if (
             askToken &&

packages/core/src/chat.ts

@@ -14,13 +14,7 @@ import {
     renderFencedVariables,
 } from "./fence"
 import { validateFencesWithSchema, validateJSONWithSchema } from "./schema"
-import {
-    CHAT_CACHE,
-    DEFAULT_MODEL,
-    DEFAULT_TEMPERATURE,
-    MAX_DATA_REPAIRS,
-    MAX_TOOL_CALLS,
-} from "./constants"
+import { CHAT_CACHE, MAX_DATA_REPAIRS, MAX_TOOL_CALLS } from "./constants"
 import { parseAnnotations } from "./annotations"
 import { isCancelError, serializeError } from "./error"
 import { details, fenceMD } from "./markdown"
@@ -489,8 +483,12 @@ export function mergeGenerationOptions(
     return {
         ...options,
         ...(runOptions || {}),
-        model: runOptions?.model ?? options?.model ?? DEFAULT_MODEL,
-        temperature: runOptions?.temperature ?? DEFAULT_TEMPERATURE,
+        model:
+            runOptions?.model ??
+            options?.model ??
+            host.defaultModelOptions.model,
+        temperature:
+            runOptions?.temperature ?? host.defaultModelOptions.temperature,
     }
 }
 
@@ -507,8 +505,8 @@ export async function executeChatSession(
 ) {
     const {
         trace,
-        model = DEFAULT_MODEL,
-        temperature = DEFAULT_TEMPERATURE,
+        model = host.defaultModelOptions.model,
+        temperature = host.defaultModelOptions.temperature,
         topP,
         maxTokens,
         seed,