Skip to content

fix: update e2e tests to use --runtime instead of renamed --agent flag #723

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Hweinstock wants to merge 1 commit into
base: main
Choose a base branch
from
+10 −10
Draft

fix: update e2e tests to use --runtime instead of renamed --agent flag #723

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion e2e-tests/byo-custom-jwt.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -239,7 +239,7 @@ describe.sequential('e2e: BYO agent with CUSTOM_JWT auth', () => {

// The CLI uses SigV4 by default — a CUSTOM_JWT runtime should reject it
const result = await runLocalCLI(
['invoke', '--prompt', 'Say hello', '--agent', agentName, '--json'],
['invoke', '--prompt', 'Say hello', '--runtime', agentName, '--json'],
projectPath
);

Expand Down
10 changes: 5 additions & 5 deletions e2e-tests/e2e-helper.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ export function createE2ESuite(cfg: E2EConfig) {
await retry(
async () => {
const result = await runAgentCoreCLI(
['invoke', '--prompt', 'Say hello', '--agent', agentName, '--json'],
['invoke', '--prompt', 'Say hello', '--runtime', agentName, '--json'],
projectPath
);

Expand Down Expand Up @@ -205,7 +205,7 @@ export function createE2ESuite(cfg: E2EConfig) {
async () => {
expect(runtimeId, 'Runtime ID should have been extracted from status').toBeTruthy();

const result = await run(['status', '--agent-runtime-id', runtimeId, '--json']);
const result = await run(['status', '--runtime-id', runtimeId, '--json']);

expect(result.exitCode, `Runtime lookup failed: ${result.stderr}`).toBe(0);

Expand All @@ -227,7 +227,7 @@ export function createE2ESuite(cfg: E2EConfig) {
await retry(
async () => {
// --since 1h triggers search mode (avoids live tail)
const result = await run(['logs', '--agent', agentName, '--since', '1h', '--json']);
const result = await run(['logs', '--runtime', agentName, '--since', '1h', '--json']);

expect(result.exitCode, `Logs failed: ${result.stderr}`).toBe(0);

Expand All @@ -254,7 +254,7 @@ export function createE2ESuite(cfg: E2EConfig) {
'logs supports level filtering',
async () => {
// --level error should succeed even if no error-level logs exist
const result = await run(['logs', '--agent', agentName, '--since', '1h', '--level', 'error', '--json']);
const result = await run(['logs', '--runtime', agentName, '--since', '1h', '--level', 'error', '--json']);

expect(result.exitCode, `Logs --level failed: ${result.stderr}`).toBe(0);
},
Expand All @@ -267,7 +267,7 @@ export function createE2ESuite(cfg: E2EConfig) {
// traces list has no --json flag — verify exit code and non-empty output
await retry(
async () => {
const result = await run(['traces', 'list', '--agent', agentName, '--since', '1h']);
const result = await run(['traces', 'list', '--runtime', agentName, '--since', '1h']);

expect(result.exitCode, `Traces list failed (stderr: ${result.stderr})`).toBe(0);
expect(result.stdout.length, 'Traces list should produce output').toBeGreaterThan(0);
Expand Down
8 changes: 4 additions & 4 deletions e2e-tests/evals-lifecycle.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ describe.sequential('e2e: evaluations lifecycle', () => {
'online-eval',
'--name',
onlineEvalName,
'--agent',
'--runtime',
agentName,
'--evaluator',
evalName,
Expand Down Expand Up @@ -118,7 +118,7 @@ describe.sequential('e2e: evaluations lifecycle', () => {
async () => {
await retry(
async () => {
const result = await run(['invoke', '--prompt', 'Say hello', '--agent', agentName, '--json']);
const result = await run(['invoke', '--prompt', 'Say hello', '--runtime', agentName, '--json']);
expect(result.exitCode, `Invoke failed: ${result.stderr}`).toBe(0);
const json = parseJsonOutput(result.stdout) as { success: boolean };
expect(json.success).toBe(true);
Expand All @@ -138,7 +138,7 @@ describe.sequential('e2e: evaluations lifecycle', () => {
const result = await run([
'run',
'eval',
'--agent',
'--runtime',
agentName,
'--evaluator',
'Builtin.Faithfulness',
Expand All @@ -162,7 +162,7 @@ describe.sequential('e2e: evaluations lifecycle', () => {
it.skipIf(!canRun)(
'eval history shows the completed run',
async () => {
const result = await run(['evals', 'history', '--agent', agentName, '--json']);
const result = await run(['evals', 'history', '--runtime', agentName, '--json']);
expect(result.exitCode, `Evals history failed: ${result.stderr}`).toBe(0);
const json = parseJsonOutput(result.stdout) as Record<string, unknown> & { runs: unknown[] };
expect(json).toHaveProperty('success', true);
Expand Down
Loading