Skip to content
Merged
2 changes: 1 addition & 1 deletion apps/vscode-e2e/fixtures/task-hello-world.json
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
"toolCalls": [
{
"name": "attempt_completion",
"arguments": "{\"result\":\"My name is Roo! I'm your AI coding assistant, here to help you with development tasks.\"}",
"arguments": "{\"result\":\"My name is Zoo! I'm your AI coding assistant, here to help you with development tasks.\"}",
"id": "call_task_hello_world_001"
}
]
Expand Down
15 changes: 15 additions & 0 deletions apps/vscode-e2e/src/fixtures/fixture-utils.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
import type { ChatCompletionRequest, ChatMessage } from "@copilotkit/aimock"

export function toolResultContains(req: ChatCompletionRequest, toolCallId: string, expected: string[]) {
const messages = Array.isArray(req?.messages) ? req.messages : []
const toolMessage = messages.find(
(message: ChatMessage) => message?.role === "tool" && message.tool_call_id === toolCallId,
)

const content = toolMessage?.content
if (typeof content !== "string") {
return false
}

return expected.every((text) => content.includes(text))
}
87 changes: 87 additions & 0 deletions apps/vscode-e2e/src/fixtures/list-files.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
import { LLMock } from "@copilotkit/aimock"

import { toolResultContains } from "./fixture-utils"

type ListFilesFixture = {
userMessagePattern: string
toolName: string
arguments: string
toolCallId: string
expected: string[]
result: string
id: string
}

export function addListFilesResultFixtures(mock: InstanceType<typeof LLMock>) {
const fixtures: ListFilesFixture[] = [
{
userMessagePattern: "without recursing into subdirectories",
toolName: "list_files",
arguments: '{"path":"list-files-tool-fixture","recursive":false}',
toolCallId: "call_list_files_non_recursive_001",
expected: ["root-file-1.txt", ".hidden-file", "nested/"],
result: "The non-recursive listing for `list-files-tool-fixture` includes `root-file-1.txt`, `root-file-2.js`, `config.yaml`, `README.md`, `.hidden-file`, and the `nested/` directory.",
Comment thread
roomote[bot] marked this conversation as resolved.
id: "call_list_files_non_recursive_002",
},
{
userMessagePattern: "deep-nested-file.ts is included",
toolName: "list_files",
arguments: '{"path":"list-files-tool-fixture","recursive":true}',
toolCallId: "call_list_files_recursive_001",
expected: ["nested/", "nested/deep/", "deep-nested-file.ts"],
result: "The recursive listing for `list-files-tool-fixture` reached the nested structure and includes `nested/`, `nested/deep/`, and `deep-nested-file.ts`.",
id: "call_list_files_recursive_002",
},
{
userMessagePattern: "path='list-files-symlink-fixture'",
toolName: "list_files",
arguments: '{"path":"list-files-symlink-fixture","recursive":false}',
toolCallId: "call_list_files_symlink_001",
expected: ["link-to-file.txt", "source/"],
result: "The symlink fixture listing shows the original `source/` directory and its `source-file.txt`, plus the symlink entry `link-to-file.txt` in `list-files-symlink-fixture`.",
id: "call_list_files_symlink_002",
},
{
userMessagePattern: "confirm whether list-files-tool-fixture or list-files-symlink-fixture is present",
toolName: "list_files",
arguments: '{"path":".","recursive":false}',
toolCallId: "call_list_files_workspace_root_001",
expected: ["list-files-tool-fixture/"],
result: "The workspace root currently contains the `list-files-tool-fixture/` and `list-files-symlink-fixture/` test directories.",
id: "call_list_files_workspace_root_002",
},
]

for (const fixture of fixtures) {
mock.addFixture({
match: {
userMessage: new RegExp(fixture.userMessagePattern),
},
response: {
toolCalls: [
{
name: fixture.toolName,
arguments: fixture.arguments,
id: fixture.toolCallId,
},
],
},
})

mock.addFixture({
match: {
toolCallId: fixture.toolCallId,
Comment thread
roomote[bot] marked this conversation as resolved.
predicate: (req) => toolResultContains(req, fixture.toolCallId, fixture.expected),
},
response: {
toolCalls: [
{
name: "attempt_completion",
arguments: JSON.stringify({ result: fixture.result }),
id: fixture.id,
},
],
},
})
}
}
144 changes: 144 additions & 0 deletions apps/vscode-e2e/src/fixtures/search-files.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
import { LLMock } from "@copilotkit/aimock"

import { toolResultContains } from "./fixture-utils"

type SearchFilesFixture = {
userMessagePattern: string
toolName: string
arguments: string
toolCallId: string
expected: string[]
result: string
id: string
}

export function addSearchFilesResultFixtures(mock: InstanceType<typeof LLMock>) {
const fixtures: SearchFilesFixture[] = [
{
userMessagePattern: "JavaScript function declarations",
toolName: "search_files",
arguments: '{"path":"search-files-tool-fixture","regex":"function\\\\s+\\\\w+"}',
toolCallId: "call_search_files_functions_001",
expected: [
"# search-files-tool-fixture/search-fixture.js",
"function calculateTotal(items) {",
"function validateUser(user) {",
],
result: "The function search found declarations including `calculateTotal`, `validateUser`, and `formatCurrency`.",
id: "call_search_files_functions_002",
},
{
userMessagePattern: "TODO comments using the regex TODO",
toolName: "search_files",
arguments: '{"path":"search-files-tool-fixture","regex":"TODO.*"}',
toolCallId: "call_search_files_todo_001",
expected: [
"# search-files-tool-fixture/search-fixture.js",
"// TODO: Add more validation functions",
"// TODO: Implement user fetching",
],
result: "The TODO search found matching TODO entries in the fixture files, including the validation and user-fetching notes.",
id: "call_search_files_todo_002",
},
{
userMessagePattern: "TypeScript interfaces you find",
toolName: "search_files",
arguments: '{"path":"search-files-tool-fixture","regex":"interface\\\\s+\\\\w+","file_pattern":"*.ts"}',
toolCallId: "call_search_files_typescript_001",
expected: ["# search-files-tool-fixture/search-fixture.ts", "interface User {", "interface Product {"],
result: "The TypeScript-only search found the `User` and `Product` interface definitions.",
id: "call_search_files_typescript_002",
},
{
userMessagePattern: "JSON configuration keys",
toolName: "search_files",
arguments: '{"path":"search-files-tool-fixture","regex":"\\"\\\\w+\\":\\\\s*","file_pattern":"*.json"}',
toolCallId: "call_search_files_json_001",
expected: ["# search-files-tool-fixture/search-config.json", '"name": "test-app",', '"dependencies": {'],
result: "The JSON search found configuration keys such as `name`, `version`, and `dependencies` in `search-config.json`.",
id: "call_search_files_json_002",
},
{
userMessagePattern: "formatCurrency and debounce",
toolName: "search_files",
arguments: '{"path":"search-files-tool-fixture","regex":"function\\\\s+(format|debounce)"}',
toolCallId: "call_search_files_nested_001",
expected: [
"# search-files-tool-fixture/nested/nested-search.js",
"function formatCurrency(amount) {",
"function debounce(func, wait) {",
],
result: "The nested-directory search found the utility functions `formatCurrency` and `debounce`.",
id: "call_search_files_nested_002",
},
{
userMessagePattern: "import and export statements",
toolName: "search_files",
arguments: '{"path":"search-files-tool-fixture","regex":"(import|export).*","file_pattern":"*.{js,ts}"}',
toolCallId: "call_search_files_complex_regex_001",
expected: [
"# search-files-tool-fixture/search-fixture.js",
"export { calculateTotal, validateUser }",
"module.exports = { formatCurrency, debounce }",
],
result: "The import/export search found the `export` statement in the JavaScript fixture module.",
id: "call_search_files_complex_regex_002",
},
{
userMessagePattern: "nonExistentPattern12345 and report that there are no matches",
toolName: "search_files",
arguments: '{"path":"search-files-tool-fixture","regex":"nonExistentPattern12345"}',
toolCallId: "call_search_files_no_match_001",
expected: ["No results found"],
result: "No matches were found for `nonExistentPattern12345` in the search fixture directory.",
id: "call_search_files_no_match_002",
},
{
userMessagePattern: "TypeScript class definitions and async methods",
toolName: "search_files",
arguments:
'{"path":"search-files-tool-fixture","regex":"(class\\\\s+\\\\w+|async\\\\s+\\\\w+)","file_pattern":"*.ts"}',
toolCallId: "call_search_files_class_method_001",
expected: [
"# search-files-tool-fixture/search-fixture.ts",
"class UserService {",
"async getUser(id: number): Promise<User> {",
],
result: "The class-and-method search found `UserService` and its async `getUser` method in the TypeScript fixture.",
id: "call_search_files_class_method_002",
},
]

for (const fixture of fixtures) {
mock.addFixture({
match: {
userMessage: new RegExp(fixture.userMessagePattern),
},
response: {
toolCalls: [
{
name: fixture.toolName,
arguments: fixture.arguments,
id: fixture.toolCallId,
},
],
},
})

mock.addFixture({
match: {
toolCallId: fixture.toolCallId,
Comment thread
roomote[bot] marked this conversation as resolved.
Comment thread
roomote[bot] marked this conversation as resolved.
predicate: (req) => toolResultContains(req, fixture.toolCallId, fixture.expected),
},
response: {
toolCalls: [
{
name: "attempt_completion",
arguments: JSON.stringify({ result: fixture.result }),
id: fixture.id,
},
],
},
})
}
}
4 changes: 4 additions & 0 deletions apps/vscode-e2e/src/runTest.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,9 @@ import * as fs from "fs/promises"
import { runTests } from "@vscode/test-electron"
import { LLMock } from "@copilotkit/aimock"

import { addListFilesResultFixtures } from "./fixtures/list-files"
import { addReadFileResultFixtures } from "./fixtures/read-file"
import { addSearchFilesResultFixtures } from "./fixtures/search-files"

function getCliFlagValue(flag: string) {
return process.argv.find((arg, index) => process.argv[index - 1] === flag)
Expand Down Expand Up @@ -77,7 +79,9 @@ async function main() {
mock.loadFixtureDir(fixturesDir)

if (!isRecord) {
addListFilesResultFixtures(mock)
addReadFileResultFixtures(mock)
addSearchFilesResultFixtures(mock)

// The modes test (switch_mode → ask) triggers a second API call whose last
// user message starts with <environment_details> directly — no <user_message>
Expand Down
12 changes: 10 additions & 2 deletions apps/vscode-e2e/src/suite/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import Mocha from "mocha"
import { glob } from "glob"
import * as vscode from "vscode"

import type { RooCodeAPI, RooCodeEventName } from "@roo-code/types"
import { RooCodeEventName, type RooCodeAPI } from "@roo-code/types"

import { waitFor } from "./utils"

Expand Down Expand Up @@ -33,12 +33,20 @@ export async function run() {

// Automatically approve completion_result asks so tests don't stall waiting
// for a button that the webview routes to "start new task" rather than "yes".
api.on("message" as RooCodeEventName.Message, ({ message }) => {
api.on(RooCodeEventName.Message, ({ message }) => {
if (message.type === "ask" && message.ask === "completion_result") {
api.approveCurrentAsk()
}
})

if (!aimockUrl) {
api.on(RooCodeEventName.Message, ({ message }) => {
if (message.type === "say" && !message.partial) {
console.log(`[say:${message.say}]`, message.text?.slice(0, 300))
}
})
}

globalThis.api = api

const mochaOptions: Mocha.MochaOptions = {
Expand Down
4 changes: 2 additions & 2 deletions apps/vscode-e2e/src/suite/task.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,9 @@ suite("Roo Code Task", function () {

assert.ok(
!!messages.find(
({ say, text }) => (say === "completion_result" || say === "text") && text?.includes("My name is Roo"),
({ say, text }) => (say === "completion_result" || say === "text") && text?.includes("My name is Zoo"),
),
`Completion should include "My name is Roo"`,
`Completion should include "My name is Zoo"`,
)
})
})
Loading
Loading