promptfoo 0.62 (#517)

* upgrade promptfoo reference * don't open in tab, generate URL pointing to local server * typos * upgrade promptfoo * cleanup * cleanup * don't open view page
microsoft · Jun 6, 2024 · f7f7373 · f7f7373
1 parent 926940a
commit f7f7373
Show file tree

Hide file tree

Showing 9 changed files with 151 additions and 144 deletions.
diff --git a/.github/workflows/ollama.yml b/.github/workflows/ollama.yml
@@ -41,5 +41,5 @@ jobs:
               run: docker run -d -v ollama:/root/.ollama -p 11434:11434 --name ollama ollama/ollama
             - name: run summarize-ollama-phi3
               run: yarn test:summarize --model ollama:phi3 --out ./temp
-            - name: run summarize-links-phi3
+            - name: test summarize-links-phi3
               run: yarn test:scripts summarize-link --models ollama:phi3 --out ./temp
diff --git a/docs/src/content/docs/reference/cli/commands.md b/docs/src/content/docs/reference/cli/commands.md
@@ -115,7 +115,7 @@ Options:
   -td, --test-delay <string>          delay between tests in seconds
   --no-cache                          disable LLM result cache
   -v, --verbose                       verbose output
-  -pv, --promptfoo-version [version]  propmtfoo version, default is ^0.61.0
+  -pv, --promptfoo-version [version]  promptfoo version, default is ^0.62.1
   -os, --out-summary <file>           append output summary in file
   -h, --help                          display help for command
 ```

diff --git a/packages/cli/package.json b/packages/cli/package.json
@@ -25,7 +25,7 @@
   "dependencies": {
     "dockerode": "^4.0.2",
     "pdfjs-dist": "4.3.136",
-    "promptfoo": "^0.61.0",
+    "promptfoo": "^0.62.1",
     "tree-sitter-wasms": "^0.1.11",
     "typescript": "5.4.5",
     "web-tree-sitter": "^0.22.2",
@@ -62,7 +62,7 @@
     "mammoth": "^1.7.2",
     "memorystream": "^0.3.1",
     "node-sarif-builder": "^3.1.0",
-    "openai": "^4.48.1",
+    "openai": "^4.48.3",
     "ora": "^8.0.1",
     "pretty-bytes": "^6.1.1",
     "prompts": "^2.4.2",

diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts
@@ -208,7 +208,7 @@ export async function cli() {
         .option("-v, --verbose", "verbose output")
         .option(
             "-pv, --promptfoo-version [version]",
-            `propmtfoo version, default is ${PROMPTFOO_VERSION}`
+            `promptfoo version, default is ${PROMPTFOO_VERSION}`
         )
         .option("-os, --out-summary <file>", "append output summary in file")
         .action(scriptsTest)

diff --git a/packages/core/package.json b/packages/core/package.json
@@ -46,7 +46,7 @@
     "mime-types": "^2.1.35",
     "minimatch": "^9.0.4",
     "minisearch": "^6.3.0",
-    "openai": "^4.48.1",
+    "openai": "^4.48.3",
     "parse-diff": "^0.11.1",
     "prettier": "^3.3.1",
     "pretty-bytes": "^6.1.1",

diff --git a/packages/core/src/promptrunner.ts b/packages/core/src/promptrunner.ts
@@ -1,23 +1,21 @@
 import { executeChatSession } from "./chat"
 import { Fragment, Project, PromptScript } from "./ast"
 import { stringToPos } from "./parser"
-import { arrayify, assert, logVerbose, relativePath, unique } from "./util"
+import { arrayify, assert, logVerbose, relativePath } from "./util"
 import { staticVars } from "./template"
 import { host } from "./host"
 import { applyLLMDiff, applyLLMPatch, parseLLMDiffs } from "./diff"
 import { MarkdownTrace } from "./trace"
 import { applyChangeLog, parseChangeLogs } from "./changelog"
 import { CORE_VERSION } from "./version"
-import { expandFiles, fileExists, readText, tryReadText } from "./fs"
+import { expandFiles, fileExists, readText } from "./fs"
 import { CSVToMarkdown } from "./csv"
 import { GenerationOptions } from "./promptcontext"
 import { traceCliArgs } from "./clihelp"
 import { GenerationResult, expandTemplate } from "./expander"
 import { resolveLanguageModel, resolveModelConnectionInfo } from "./models"
 import { RequestError, errorMessage } from "./error"
-import { createFetch } from "./fetch"
 import { unquote } from "./fence"
-import { HTTPS_REGEX } from "./constants"
 import { parsePromptParameters } from "./parameters"
 import { resolveFileContent } from "./file"
 

diff --git a/packages/core/src/server/messages.ts b/packages/core/src/server/messages.ts
@@ -63,7 +63,7 @@ export interface PromptScriptTestRun extends RequestMessage {
 
 export interface PromptScriptTestResult extends ResponseStatus {
     script: string
-    value?: any /** OutputFile */
+    value?: { evalId: string } /** OutputFile */
 }
 
 export interface PromptScriptTestRunResponse extends ResponseStatus {

diff --git a/packages/vscode/src/testcontroller.ts b/packages/vscode/src/testcontroller.ts
@@ -2,6 +2,8 @@ import * as vscode from "vscode"
 import { ExtensionState } from "./state"
 import {
     CHANGE,
+    EMOJI_FAIL,
+    EMOJI_SUCCESS,
     ICON_LOGO_NAME,
     PROMPTFOO_CACHE_PATH,
     PROMPTFOO_CONFIG_DIR,
@@ -94,7 +96,7 @@ export async function activateTestController(state: ExtensionState) {
                 return
             }
 
-            startTestViewer()
+            const serverUrl = await startTestViewer()
             await state.host.server.client.init()
             try {
                 for (const { script, test } of scripts) {
@@ -105,7 +107,13 @@ export async function activateTestController(state: ExtensionState) {
                     }
                     run.started(test)
                     const res = await state.host.server.client.runTest(script)
-                    run.appendOutput(JSON.stringify(res), undefined, test)
+                    for (const r of res.value || []) {
+                        run.appendOutput(
+                            `${r.ok ? EMOJI_SUCCESS : EMOJI_FAIL} ${r.script} ${errorMessage(r.error) || ""} ${serverUrl}/eval?evalId=${encodeURIComponent(r.value?.evalId)}`,
+                            undefined,
+                            test
+                        )
+                    }
                     if (res.error)
                         run.failed(
                             test,
@@ -135,9 +143,10 @@ export async function activateTestController(state: ExtensionState) {
     }
 }
 
-export async function startTestViewer() {
+async function startTestViewer() {
     const name = "Promptfoo View"
     const port = PROMPTFOO_REMOTE_API_PORT
+    const serverUrl = `http://127.0.0.1:${port}`
     if (!vscode.window.terminals.find((t) => t.name === name)) {
         // show results
         const terminal = vscode.window.createTerminal({
@@ -153,8 +162,8 @@ export async function startTestViewer() {
         })
         const promptfooVersion = PROMPTFOO_VERSION
         terminal.sendText(
-            `npx --yes promptfoo@${promptfooVersion} view --port ${port}`
+            `npx --yes promptfoo@${promptfooVersion} view --port ${port} --no`
         )
-        await openUrlInTab(`http://127.0.0.1:${port}`)
     }
+    return serverUrl
 }