Skip to content

chore: add resilience-test suite and remove individual test #6807

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
PeterSchafer wants to merge 1 commit into
base: main
Choose a base branch
from
+269 −206
Open

chore: add resilience-test suite and remove individual test #6807

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions test/acceptance/fake-server.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ const featureFlagDefaults = (): Map<string, boolean> => {
['sbomTestReachability', false],
['useTestShimForOSCliTest', false],
['cliDotnetRuntimeResolution', false],
['isSecretsEnabled', true],
Copy link
Copy Markdown
Contributor Author

@PeterSchafer PeterSchafer May 14, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe we want to enable this only in the tests, to avoid doing this globally.

]);
};

Expand Down
103 changes: 0 additions & 103 deletions test/jest/acceptance/maintenance.spec.ts
View file
Open in desktop

This file was deleted.

268 changes: 268 additions & 0 deletions test/jest/acceptance/resilience.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,268 @@
import { fakeServer, getFirstIPv4Address } from '../../acceptance/fake-server';
import { runSnykCLI } from '../util/runSnykCLI';
import { getAvailableServerPort } from '../util/getServerPort';
import { Snyk } from '@snyk/error-catalog-nodejs-public';
import { EXIT_CODES } from '../../../src/cli/exit-codes';
import { getCliConfig, restoreCliConfig } from '../../acceptance/config-helper';

jest.setTimeout(1000 * 60);

const TIMEOUT_SECS = 5;
const GRACE_PERIOD_SECS = 5;
const SERVER_DELAY_MS = 10000;
const FAKE_ORG = '11111111-1111-1111-1111-111111111111';

// Commands that should behave consistently across all fault scenarios
const COMMANDS_UNDER_TEST = [
'test',
'code test',
'container test scratch',
'container monitor scratch',
'iac test',
'secrets test',
'monitor',
'whoami',
'auth 11111111-2222-3333-4444-555555555555',
'sbom --org=11111111-1111-1111-1111-111111111111 --format=cyclonedx1.4+json',
'container sbom scratch --format=cyclonedx1.4+json',
'sbom test --experimental --file=package.json',
'aibom test --experimental',
];

interface ScenarioContext {
server: ReturnType<typeof fakeServer>;
savedConfig?: Record<string, string>;
}

interface TestResult {
code: number;
stdout: string;
duration: number;
}

interface AssertionContext {
server: ReturnType<typeof fakeServer>;
result: TestResult;
}

interface ResilienceScenario {
name: string;
description: string;
setup: (ctx: ScenarioContext) => void | Promise<void>;
teardown?: (ctx: ScenarioContext) => void | Promise<void>;
expectedExitCode: number;
expectedErrorCode: string;
assert?: (ctx: AssertionContext) => void; // Additional scenario-specific assertions
envOverrides?: Record<string, string>;
skip?: string[]; // Commands to skip for this scenario (not yet consistent)
}

const RESILIENCE_SCENARIOS: ResilienceScenario[] = [
// Scenario 1
{
name: 'maintenance-window',
description: 'Backend in maintenance mode (503 with error catalog)',
setup: ({ server }) => {
const maintenanceErrorRes = {
jsonapi: { version: '1.0' },
errors: [new Snyk.MaintenanceWindowError('').toJsonApiErrorObject()],
description: 'Maintenance window',
};
server.setGlobalResponse(
maintenanceErrorRes,
parseInt(maintenanceErrorRes.errors[0].status),
);
},
expectedExitCode: EXIT_CODES.EX_TEMPFAIL,
expectedErrorCode: 'SNYK-0099',
assert: ({ server }) => {
// Verify no retries (fail fast for maintenance)
// Each snyk-request-id should appear only once - duplicates indicate retries
const requests = server.getRequests();
const requestIdCounts = new Map<string, number>();
for (const req of requests) {
const header = req.headers?.['snyk-request-id'];
const requestId = Array.isArray(header) ? header[0] : header;
if (requestId) {
requestIdCounts.set(
requestId,
(requestIdCounts.get(requestId) ?? 0) + 1,
);
}
}
for (const count of requestIdCounts.values()) {
expect(count).toBe(1);
}
},
envOverrides: {
// Enable retries to verify they are NOT used
SNYK_MAX_ATTEMPTS: '10',
},
},

// Scenario 2
{
name: 'timeout',
description: 'CLI times out before command finishes',
setup: ({ server }) => {
server.setResponseDelay(SERVER_DELAY_MS);
},
expectedExitCode: EXIT_CODES.EX_UNAVAILABLE,
expectedErrorCode: 'SNYK-CLI-0026',
assert: ({ result }) => {
// Verify timeout occurred within expected bounds
expect(result.duration).toBeGreaterThanOrEqual(TIMEOUT_SECS * 1000);
expect(result.duration).toBeLessThan(
(TIMEOUT_SECS + GRACE_PERIOD_SECS) * 1000,
);
},
envOverrides: {
SNYK_TIMEOUT_SECS: String(TIMEOUT_SECS),
},
skip: ['container sbom scratch'],
},

// Scenario 3
{
name: 'unauthorized-401',
description: 'Backend returns 401 Unauthorized',
setup: ({ server }) => {
server.setGlobalResponse(
{
jsonapi: { version: '1.0' },
errors: [new Snyk.UnauthorisedError('').toJsonApiErrorObject()],
},
401,
);
},
expectedExitCode: EXIT_CODES.ERROR,
expectedErrorCode: 'SNYK-0005',
skip: [
'container sbom scratch',
'container test scratch',
'container monitor scratch',
'iac test',
'secrets test',
'auth', // auth doesn't need to
],
},

// Scenario 4
{
name: 'mid-execution-maintenance',
description: 'Backend enters maintenance after initial successful requests',
setup: ({ server }) => {
const maintenanceErrorRes = {
jsonapi: { version: '1.0' },
errors: [new Snyk.MaintenanceWindowError('').toJsonApiErrorObject()],
description: 'Maintenance window',
};

// First request succeeds, subsequent requests hit maintenance
server.setNextStatusCode(200);
server.setNextStatusCode(200);
server.setNextStatusCode(200);
server.setNextStatusCode(200);
server.setGlobalResponse(
maintenanceErrorRes,
parseInt(maintenanceErrorRes.errors[0].status),
);
},
expectedExitCode: EXIT_CODES.EX_TEMPFAIL,
expectedErrorCode: 'SNYK-0099',
skip: [
'whoami', // Single-request commands won't hit the failure
'auth', // Single-request commands won't hit the failure
'container monitor scratch',
],
},
];

function shouldSkip(scenario: ResilienceScenario, command: string): boolean {
if (!scenario.skip) return false;
return scenario.skip.some((skip) => command.startsWith(skip));
}

describe('Resilience - Consistent CLI Behavior', () => {
let server: ReturnType<typeof fakeServer>;
let baseEnv: Record<string, string>;

beforeAll(async () => {
const ipAddr = getFirstIPv4Address();
const port = await getAvailableServerPort(process);
const baseApi = '/api/v1';

baseEnv = {
...process.env,
SNYK_API: 'http://' + ipAddr + ':' + port + baseApi,
SNYK_TOKEN: '123456789',
SNYK_HTTP_PROTOCOL_UPGRADE: '0',
SNYK_CFG_ORG: FAKE_ORG,
};

server = fakeServer(baseApi, baseEnv.SNYK_TOKEN);
await server.listenPromise(port);
});

afterEach(() => {
server.restore();
});

afterAll(async () => {
await server.closePromise();
});

describe.each(RESILIENCE_SCENARIOS)(
'$name: $description',
(scenario: ResilienceScenario) => {
const commandsToRun = COMMANDS_UNDER_TEST.filter(
(cmd) => !shouldSkip(scenario, cmd),
);
const commandsToSkip = COMMANDS_UNDER_TEST.filter((cmd) =>
shouldSkip(scenario, cmd),
);

if (commandsToSkip.length > 0) {
it.skip.each(commandsToSkip)('"%s" (not yet consistent)', () => {});
}

it.each(commandsToRun)('"%s"', async (command) => {
const ctx: ScenarioContext = { server };
const requiresConfigRestore = command.startsWith('auth');

try {
if (requiresConfigRestore) {
ctx.savedConfig = await getCliConfig();
}

await scenario.setup(ctx);
const env = { ...baseEnv, ...scenario.envOverrides };

const startTime = Date.now();
const { code, stdout } = await runSnykCLI(command, { env });
const duration = Date.now() - startTime;

// Common assertions
expect(code).toEqual(scenario.expectedExitCode);
expect(stdout).toContain(scenario.expectedErrorCode);

// Scenario-specific assertions
if (scenario.assert) {
scenario.assert({
server,
result: { code, stdout, duration },
});
}
} finally {
server.restore();
if (scenario.teardown) {
await scenario.teardown(ctx);
}
if (requiresConfigRestore && ctx.savedConfig) {
await restoreCliConfig(ctx.savedConfig);
}
}
});
},
);
});
Loading
Loading