diff --git a/src/compiler/base/nodes/tags/step.test.ts b/src/compiler/base/nodes/tags/step.test.ts index d0e5c7c..3ee58d6 100644 --- a/src/compiler/base/nodes/tags/step.test.ts +++ b/src/compiler/base/nodes/tags/step.test.ts @@ -1,12 +1,12 @@ import CompileError from '$promptl/error/error' -import { complete, getExpectedError } from "$promptl/compiler/test/helpers"; -import { removeCommonIndent } from "$promptl/compiler/utils"; -import { Chain } from "$promptl/index"; -import { describe, expect, it, vi } from "vitest"; +import { complete } from '$promptl/compiler/test/helpers' +import { removeCommonIndent } from '$promptl/compiler/utils' +import { Chain } from '$promptl/index' +import { describe, expect, it, vi } from 'vitest' -describe("step tags", async () => { - it("does not create a variable from response if not specified", async () => { - const mock = vi.fn(); +describe('step tags', async () => { + it('does not create a variable from response if not specified', async () => { + const mock = vi.fn() const prompt = removeCommonIndent(` Ensure truthfulness of the following statement, give a reason and a confidence score. @@ -15,18 +15,22 @@ describe("step tags", async () => { Now correct the statement if it is not true. - `); + `) - const chain = new Chain({ prompt, parameters: { mock }}); - await complete({ chain, callback: async () => ` + const chain = new Chain({ prompt, parameters: { mock } }) + await complete({ + chain, + callback: async () => + ` The statement is not true because it is fake. My confidence score is 100. - `.trim()}); + `.trim(), + }) - expect(mock).not.toHaveBeenCalled(); - }); + expect(mock).not.toHaveBeenCalled() + }) - it("creates a text variable from response if specified", async () => { - const mock = vi.fn(); + it('creates a text variable from response if specified', async () => { + const mock = vi.fn() const prompt = removeCommonIndent(` Ensure truthfulness of the following statement, give a reason and a confidence score. @@ -36,18 +40,24 @@ describe("step tags", async () => { {{ mock(analysis) }} Now correct the statement if it is not true. - `); + `) - const chain = new Chain({ prompt, parameters: { mock }}); - await complete({ chain, callback: async () => ` + const chain = new Chain({ prompt, parameters: { mock } }) + await complete({ + chain, + callback: async () => + ` The statement is not true because it is fake. My confidence score is 100. - `.trim()}); + `.trim(), + }) - expect(mock).toHaveBeenCalledWith("The statement is not true because it is fake. My confidence score is 100."); - }); + expect(mock).toHaveBeenCalledWith( + 'The statement is not true because it is fake. My confidence score is 100.', + ) + }) - it("creates an object variable from response if specified and schema is provided", async () => { - const mock = vi.fn(); + it('creates an object variable from response if specified and schema is provided', async () => { + const mock = vi.fn() const prompt = removeCommonIndent(` Ensure truthfulness of the following statement, give a reason and a confidence score. @@ -59,27 +69,33 @@ describe("step tags", async () => { Correct the statement taking into account the reason: '{{ analysis.reason }}'. {{ endif }} - `); + `) - const chain = new Chain({ prompt, parameters: { mock }}); - const { messages } = await complete({ chain, callback: async () => ` + const chain = new Chain({ prompt, parameters: { mock } }) + const { messages } = await complete({ + chain, + callback: async () => + ` { "truthful": false, "reason": "It is fake", "confidence": 100 } - `.trim()}); + `.trim(), + }) expect(mock).toHaveBeenCalledWith({ truthful: false, - reason: "It is fake", - confidence: 100 - }); - expect(messages[2]!.content).toEqual("Correct the statement taking into account the reason: 'It is fake'."); - }); + reason: 'It is fake', + confidence: 100, + }) + expect(messages[2]!.content).toEqual( + "Correct the statement taking into account the reason: 'It is fake'.", + ) + }) - it("fails creating an object variable from response if specified and schema is provided but response is invalid", async () => { - const mock = vi.fn(); + it('fails creating an object variable from response if specified and schema is provided but response is invalid', async () => { + const mock = vi.fn() const prompt = removeCommonIndent(` Ensure truthfulness of the following statement, give a reason and a confidence score. @@ -91,19 +107,29 @@ describe("step tags", async () => { Correct the statement taking into account the reason: '{{ analysis.reason }}'. {{ endif }} - `); + `) - const chain = new Chain({ prompt, parameters: { mock }}); - const error = await getExpectedError(() => complete({ chain, callback: async () => ` + const chain = new Chain({ prompt, parameters: { mock } }) + let error: CompileError + try { + await complete({ + chain, + callback: async () => + ` Bad JSON. - `.trim()}), CompileError) - expect(error.code).toBe('invalid-step-response-format') + `.trim(), + }) + } catch (e) { + error = e as CompileError + expect(e).toBeInstanceOf(CompileError) + } - expect(mock).not.toHaveBeenCalled(); - }); + expect(error!.code).toBe('invalid-step-response-format') + expect(mock).not.toHaveBeenCalled() + }) - it("creates a raw variable from response if specified", async () => { - const mock = vi.fn(); + it('creates a raw variable from response if specified', async () => { + const mock = vi.fn() const prompt = removeCommonIndent(` Ensure truthfulness of the following statement, give a reason and a confidence score. @@ -113,21 +139,25 @@ describe("step tags", async () => { {{ mock(analysis) }} Now correct the statement if it is not true. - `); + `) - const chain = new Chain({ prompt, parameters: { mock }}); - await complete({ chain, callback: async () => ` + const chain = new Chain({ prompt, parameters: { mock } }) + await complete({ + chain, + callback: async () => + ` The statement is not true because it is fake. My confidence score is 100. - `.trim()}); + `.trim(), + }) expect(mock).toHaveBeenCalledWith({ - role: "assistant", + role: 'assistant', content: [ { - type: "text", - text: "The statement is not true because it is fake. My confidence score is 100.", + type: 'text', + text: 'The statement is not true because it is fake. My confidence score is 100.', }, ], - }); - }); -}); + }) + }) +}) diff --git a/src/compiler/scan.test.ts b/src/compiler/scan.test.ts index 0ec23fa..f18c8a5 100644 --- a/src/compiler/scan.test.ts +++ b/src/compiler/scan.test.ts @@ -762,9 +762,16 @@ describe('syntax errors', async () => { const metadata = await scan({ prompt }) - expect(metadata.errors).toEqual([ - new CompileError('Tool messages must have an id attribute'), - ]) + expect(metadata.errors.length).toBe(1) + const error = metadata.errors[0]! + expect(error.name).toBe('CompileError') + expect(error.code).toBe('tool-message-without-id') + expect(error.message).toBe('Tool messages must have an id attribute') + expect(error.startIndex).toBe(0) + expect(error.endIndex).toBe(19) + expect(error.frame).toEqual( + '1: Tool 1\n\n ^~~~~~~~~~~~~~~~~~~', + ) }) it('throw error if tool does not have name', async () => { @@ -775,9 +782,15 @@ describe('syntax errors', async () => { const metadata = await scan({ prompt }) expect(metadata.errors).toEqual([ - new CompileError( - 'Tool messages must have a name attribute equal to the tool name used in tool-call', - ), + new CompileError({ + message: + 'Tool messages must have a name attribute equal to the tool name used in tool-call', + startIndex: 0, + endIndex: 21, + name: 'Tool 1', + code: 'tool-missing-name', + frame: expect.any(String), + }), ]) }) }) diff --git a/src/error/error.ts b/src/error/error.ts index 01f6796..18e8d60 100644 --- a/src/error/error.ts +++ b/src/error/error.ts @@ -16,13 +16,51 @@ type CompileErrorProps = { } export default class CompileError extends Error { - code?: string + startIndex: number + endIndex: number + name: string + code: string + frame: string start?: Position end?: Position pos?: number - frame?: string fragment?: Fragment + constructor({ + message, + name, + code, + startIndex, + endIndex, + start, + end, + frame, + fragment, + }: { + message: string + name: string + code: string + startIndex: number + endIndex: number + frame: string + start?: Position + end?: Position + fragment?: Fragment + }) { + super(message) + + this.name = name + this.code = code + this.startIndex = startIndex + // Legacy alias for startIndex + this.pos = this.startIndex + this.endIndex = endIndex + this.start = start + this.end = end + this.frame = frame + this.fragment = fragment + } + toString() { if (!this.start) return this.message return `${this.message} (${this.start.line}:${this.start.column})\n${this.frame}` @@ -63,8 +101,6 @@ function getCodeFrame( } export function error(message: string, props: CompileErrorProps): never { - const error = new CompileError(message) - error.name = props.name const start = locate(props.source, props.start, { offsetLine: 1, offsetColumn: 1, @@ -73,16 +109,21 @@ export function error(message: string, props: CompileErrorProps): never { offsetLine: 1, offsetColumn: 1, }) - error.code = props.code - error.start = start - error.end = end - error.pos = props.start - error.frame = getCodeFrame( - props.source, - (start?.line ?? 1) - 1, - start?.column ?? 0, - end?.column, - ) - error.fragment = props.fragment - throw error + const endIndex = props.end ?? props.start + throw new CompileError({ + message, + name: props.name, + code: props.code, + startIndex: props.start, + endIndex, + start, + end, + frame: getCodeFrame( + props.source, + (start?.line ?? 1) - 1, + start?.column ?? 0, + end?.column, + ), + fragment: props.fragment, + }) } diff --git a/src/test/helpers.ts b/src/test/helpers.ts index aa9c2ce..41d6dc6 100644 --- a/src/test/helpers.ts +++ b/src/test/helpers.ts @@ -1,8 +1,8 @@ import { expect } from 'vitest' -export async function getExpectedError( +export async function getExpectedError( action: () => unknown, - errorClass: new () => T, + errorClass: new (...args: any[]) => T, ): Promise { try { await action()