diff --git a/test/acceptance/fake-server.ts b/test/acceptance/fake-server.ts index b5073ac470..f789a83e80 100644 --- a/test/acceptance/fake-server.ts +++ b/test/acceptance/fake-server.ts @@ -33,6 +33,7 @@ const featureFlagDefaults = (): Map => { ['sbomTestReachability', false], ['useTestShimForOSCliTest', false], ['cliDotnetRuntimeResolution', false], + ['isSecretsEnabled', true], ]); }; diff --git a/test/jest/acceptance/maintenance.spec.ts b/test/jest/acceptance/maintenance.spec.ts deleted file mode 100644 index 90fe23d29c..0000000000 --- a/test/jest/acceptance/maintenance.spec.ts +++ /dev/null @@ -1,103 +0,0 @@ -import { fakeServer, getFirstIPv4Address } from '../../acceptance/fake-server'; -import { runSnykCLI } from '../util/runSnykCLI'; -import { getAvailableServerPort } from '../util/getServerPort'; -import { Snyk } from '@snyk/error-catalog-nodejs-public'; -import { EXIT_CODES } from '../../../src/cli/exit-codes'; -import { getCliConfig, restoreCliConfig } from '../../acceptance/config-helper'; - -jest.setTimeout(1000 * 30); - -describe('maintenance error [SNYK-0099]', () => { - let server: ReturnType; - let env: Record; - let initialConfig: Record = {}; - - const maintenanceErrorRes = { - jsonapi: { version: '1.0' }, - errors: [new Snyk.MaintenanceWindowError('').toJsonApiErrorObject()], - description: 'Maintenance window', - }; - - beforeAll(async () => { - const ipAddr = getFirstIPv4Address(); - const port = await getAvailableServerPort(process); - const baseApi = '/api/v1'; - - env = { - ...process.env, - SNYK_API: 'http://' + ipAddr + ':' + port + baseApi, - SNYK_TOKEN: '123456789', - SNYK_HTTP_PROTOCOL_UPGRADE: '0', - INTERNAL_NETWORK_REQUEST_MAX_ATTEMPTS: '1', - INTERNAL_NETWORK_REQUEST_RETRY_AFTER_SECONDS: '1', - }; - server = fakeServer(baseApi, env.SNYK_TOKEN); - await new Promise((resolve) => { - server.listen(port, () => { - resolve(); - }); - }); - }); - - beforeEach(async () => { - initialConfig = await getCliConfig(); - server.setGlobalResponse( - maintenanceErrorRes, - parseInt(maintenanceErrorRes.errors[0].status), - ); - }); - - afterEach(async () => { - server.restore(); - await restoreCliConfig(initialConfig); - }); - - afterAll(async () => { - await new Promise((resolve) => { - server.close(() => { - resolve(); - }); - }); - }); - - it('does not attempt any retries', async () => { - await runSnykCLI(`test -d --log-level=trace`, { - env: { - ...env, - // apply a user configured attempts of 10 - INTERNAL_NETWORK_REQUEST_MAX_ATTEMPTS: '10', - }, - }); - - // Count how many times an endpoint was hit - const requests = server.getRequests(); - const actualNetworkAttempts = requests.filter( - (r) => r.url.includes('/test-dep-graph') || r.url.includes('/vuln/'), - ).length; - - expect(actualNetworkAttempts).toBe(1); - }); - - it.each([ - ['test'], - ['container test scratch'], - ['container monitor scratch'], - ['iac test'], - ['code test'], - ['secrets test'], - ['monitor'], - ['whoami'], - ['auth 11111111-2222-3333-4444-555555555555'], - ['sbom --org=test-org --format=cyclonedx1.4+json'], - ['container sbom scratch --format=cyclonedx1.4+json'], - ['sbom test --experimental --file=package.json'], - ['aibom test --experimental'], - ])('returns correct exit code for "%s"', async (args) => { - const { code, stdout } = await runSnykCLI(args, { - env, - }); - - expect(stdout).toContain(maintenanceErrorRes['errors'][0].code); - expect(code).toEqual(EXIT_CODES.EX_TEMPFAIL); - }); -}); diff --git a/test/jest/acceptance/resilience.spec.ts b/test/jest/acceptance/resilience.spec.ts new file mode 100644 index 0000000000..381b393fab --- /dev/null +++ b/test/jest/acceptance/resilience.spec.ts @@ -0,0 +1,268 @@ +import { fakeServer, getFirstIPv4Address } from '../../acceptance/fake-server'; +import { runSnykCLI } from '../util/runSnykCLI'; +import { getAvailableServerPort } from '../util/getServerPort'; +import { Snyk } from '@snyk/error-catalog-nodejs-public'; +import { EXIT_CODES } from '../../../src/cli/exit-codes'; +import { getCliConfig, restoreCliConfig } from '../../acceptance/config-helper'; + +jest.setTimeout(1000 * 60); + +const TIMEOUT_SECS = 5; +const GRACE_PERIOD_SECS = 5; +const SERVER_DELAY_MS = 10000; +const FAKE_ORG = '11111111-1111-1111-1111-111111111111'; + +// Commands that should behave consistently across all fault scenarios +const COMMANDS_UNDER_TEST = [ + 'test', + 'code test', + 'container test scratch', + 'container monitor scratch', + 'iac test', + 'secrets test', + 'monitor', + 'whoami', + 'auth 11111111-2222-3333-4444-555555555555', + 'sbom --org=11111111-1111-1111-1111-111111111111 --format=cyclonedx1.4+json', + 'container sbom scratch --format=cyclonedx1.4+json', + 'sbom test --experimental --file=package.json', + 'aibom test --experimental', +]; + +interface ScenarioContext { + server: ReturnType; + savedConfig?: Record; +} + +interface TestResult { + code: number; + stdout: string; + duration: number; +} + +interface AssertionContext { + server: ReturnType; + result: TestResult; +} + +interface ResilienceScenario { + name: string; + description: string; + setup: (ctx: ScenarioContext) => void | Promise; + teardown?: (ctx: ScenarioContext) => void | Promise; + expectedExitCode: number; + expectedErrorCode: string; + assert?: (ctx: AssertionContext) => void; // Additional scenario-specific assertions + envOverrides?: Record; + skip?: string[]; // Commands to skip for this scenario (not yet consistent) +} + +const RESILIENCE_SCENARIOS: ResilienceScenario[] = [ + // Scenario 1 + { + name: 'maintenance-window', + description: 'Backend in maintenance mode (503 with error catalog)', + setup: ({ server }) => { + const maintenanceErrorRes = { + jsonapi: { version: '1.0' }, + errors: [new Snyk.MaintenanceWindowError('').toJsonApiErrorObject()], + description: 'Maintenance window', + }; + server.setGlobalResponse( + maintenanceErrorRes, + parseInt(maintenanceErrorRes.errors[0].status), + ); + }, + expectedExitCode: EXIT_CODES.EX_TEMPFAIL, + expectedErrorCode: 'SNYK-0099', + assert: ({ server }) => { + // Verify no retries (fail fast for maintenance) + // Each snyk-request-id should appear only once - duplicates indicate retries + const requests = server.getRequests(); + const requestIdCounts = new Map(); + for (const req of requests) { + const header = req.headers?.['snyk-request-id']; + const requestId = Array.isArray(header) ? header[0] : header; + if (requestId) { + requestIdCounts.set( + requestId, + (requestIdCounts.get(requestId) ?? 0) + 1, + ); + } + } + for (const count of requestIdCounts.values()) { + expect(count).toBe(1); + } + }, + envOverrides: { + // Enable retries to verify they are NOT used + SNYK_MAX_ATTEMPTS: '10', + }, + }, + + // Scenario 2 + { + name: 'timeout', + description: 'CLI times out before command finishes', + setup: ({ server }) => { + server.setResponseDelay(SERVER_DELAY_MS); + }, + expectedExitCode: EXIT_CODES.EX_UNAVAILABLE, + expectedErrorCode: 'SNYK-CLI-0026', + assert: ({ result }) => { + // Verify timeout occurred within expected bounds + expect(result.duration).toBeGreaterThanOrEqual(TIMEOUT_SECS * 1000); + expect(result.duration).toBeLessThan( + (TIMEOUT_SECS + GRACE_PERIOD_SECS) * 1000, + ); + }, + envOverrides: { + SNYK_TIMEOUT_SECS: String(TIMEOUT_SECS), + }, + skip: ['container sbom scratch'], + }, + + // Scenario 3 + { + name: 'unauthorized-401', + description: 'Backend returns 401 Unauthorized', + setup: ({ server }) => { + server.setGlobalResponse( + { + jsonapi: { version: '1.0' }, + errors: [new Snyk.UnauthorisedError('').toJsonApiErrorObject()], + }, + 401, + ); + }, + expectedExitCode: EXIT_CODES.ERROR, + expectedErrorCode: 'SNYK-0005', + skip: [ + 'container sbom scratch', + 'container test scratch', + 'container monitor scratch', + 'iac test', + 'secrets test', + 'auth', // auth doesn't need to + ], + }, + + // Scenario 4 + { + name: 'mid-execution-maintenance', + description: 'Backend enters maintenance after initial successful requests', + setup: ({ server }) => { + const maintenanceErrorRes = { + jsonapi: { version: '1.0' }, + errors: [new Snyk.MaintenanceWindowError('').toJsonApiErrorObject()], + description: 'Maintenance window', + }; + + // First request succeeds, subsequent requests hit maintenance + server.setNextStatusCode(200); + server.setNextStatusCode(200); + server.setNextStatusCode(200); + server.setNextStatusCode(200); + server.setGlobalResponse( + maintenanceErrorRes, + parseInt(maintenanceErrorRes.errors[0].status), + ); + }, + expectedExitCode: EXIT_CODES.EX_TEMPFAIL, + expectedErrorCode: 'SNYK-0099', + skip: [ + 'whoami', // Single-request commands won't hit the failure + 'auth', // Single-request commands won't hit the failure + 'container monitor scratch', + ], + }, +]; + +function shouldSkip(scenario: ResilienceScenario, command: string): boolean { + if (!scenario.skip) return false; + return scenario.skip.some((skip) => command.startsWith(skip)); +} + +describe('Resilience - Consistent CLI Behavior', () => { + let server: ReturnType; + let baseEnv: Record; + + beforeAll(async () => { + const ipAddr = getFirstIPv4Address(); + const port = await getAvailableServerPort(process); + const baseApi = '/api/v1'; + + baseEnv = { + ...process.env, + SNYK_API: 'http://' + ipAddr + ':' + port + baseApi, + SNYK_TOKEN: '123456789', + SNYK_HTTP_PROTOCOL_UPGRADE: '0', + SNYK_CFG_ORG: FAKE_ORG, + }; + + server = fakeServer(baseApi, baseEnv.SNYK_TOKEN); + await server.listenPromise(port); + }); + + afterEach(() => { + server.restore(); + }); + + afterAll(async () => { + await server.closePromise(); + }); + + describe.each(RESILIENCE_SCENARIOS)( + '$name: $description', + (scenario: ResilienceScenario) => { + const commandsToRun = COMMANDS_UNDER_TEST.filter( + (cmd) => !shouldSkip(scenario, cmd), + ); + const commandsToSkip = COMMANDS_UNDER_TEST.filter((cmd) => + shouldSkip(scenario, cmd), + ); + + if (commandsToSkip.length > 0) { + it.skip.each(commandsToSkip)('"%s" (not yet consistent)', () => {}); + } + + it.each(commandsToRun)('"%s"', async (command) => { + const ctx: ScenarioContext = { server }; + const requiresConfigRestore = command.startsWith('auth'); + + try { + if (requiresConfigRestore) { + ctx.savedConfig = await getCliConfig(); + } + + await scenario.setup(ctx); + const env = { ...baseEnv, ...scenario.envOverrides }; + + const startTime = Date.now(); + const { code, stdout } = await runSnykCLI(command, { env }); + const duration = Date.now() - startTime; + + // Common assertions + expect(code).toEqual(scenario.expectedExitCode); + expect(stdout).toContain(scenario.expectedErrorCode); + + // Scenario-specific assertions + if (scenario.assert) { + scenario.assert({ + server, + result: { code, stdout, duration }, + }); + } + } finally { + server.restore(); + if (scenario.teardown) { + await scenario.teardown(ctx); + } + if (requiresConfigRestore && ctx.savedConfig) { + await restoreCliConfig(ctx.savedConfig); + } + } + }); + }, + ); +}); diff --git a/test/jest/acceptance/timeout.spec.ts b/test/jest/acceptance/timeout.spec.ts deleted file mode 100644 index 49c2271cdc..0000000000 --- a/test/jest/acceptance/timeout.spec.ts +++ /dev/null @@ -1,103 +0,0 @@ -import { fakeServer, getFirstIPv4Address } from '../../acceptance/fake-server'; -import { runSnykCLI } from '../util/runSnykCLI'; -import { getAvailableServerPort } from '../util/getServerPort'; -import { EXIT_CODES } from '../../../src/cli/exit-codes'; -import { getCliConfig, restoreCliConfig } from '../../acceptance/config-helper'; - -jest.setTimeout(1000 * 60); // 60 seconds - tests involve timeouts - -// SNYK_TIMEOUT_SECS=5, server delay=10s, grace period=3s -// Expected: CLI should timeout around 8s (5+3), definitely before server responds at 10s -const TIMEOUT_SECS = 5; -const GRACE_PERIOD_SECS = 5; -const SERVER_DELAY_MS = 10000; -const EXPECTED_MIN_MS = TIMEOUT_SECS * 1000; // At least the timeout duration -const EXPECTED_MAX_MS = (TIMEOUT_SECS + GRACE_PERIOD_SECS) * 1000; // Timeout + grace -const orgId = '11111111-2222-3333-4444-555555555555'; - -describe('timeout behavior [exit code 69]', () => { - let server: ReturnType; - let env: Record; - let initialConfig: Record = {}; - - beforeAll(async () => { - const ipAddr = getFirstIPv4Address(); - const port = await getAvailableServerPort(process); - const baseApi = '/api/v1'; - - env = { - ...process.env, - SNYK_API: 'http://' + ipAddr + ':' + port + baseApi, - SNYK_TOKEN: '123456789', - SNYK_HTTP_PROTOCOL_UPGRADE: '0', - SNYK_CFG_ORG: orgId, - // Disable retries to speed up tests - INTERNAL_NETWORK_REQUEST_MAX_ATTEMPTS: '1', - // Set a short timeout for testing (5 seconds) - SNYK_TIMEOUT_SECS: String(TIMEOUT_SECS), - }; - - server = fakeServer(baseApi, env.SNYK_TOKEN); - await server.listenPromise(port); - }); - - beforeEach(async () => { - initialConfig = await getCliConfig(); - // Set server to delay responses longer than the timeout (10s > 5s timeout) - server.setResponseDelay(SERVER_DELAY_MS); - }); - - afterEach(async () => { - server.restore(); - await restoreCliConfig(initialConfig); - }); - - afterAll(async () => { - await server.closePromise(); - }); - - it.each([ - ['code test'], - ['test'], - ['container test scratch'], - ['container monitor scratch'], - ['iac test'], - ['monitor'], - ['whoami'], - ['auth 11111111-2222-3333-4444-555555555555'], - ['sbom --format=cyclonedx1.4+json -d'], - ])( - 'returns exit code 69 (EX_UNAVAILABLE) on timeout for "%s"', - async (args) => { - const startTime = Date.now(); - const { code, stdout } = await runSnykCLI(args, { - env, - }); - const duration = Date.now() - startTime; - - console.log(stdout); - - // print duration and min and max in seconds - console.log( - `Duration: ${duration / 1000} seconds, Min: ${EXPECTED_MIN_MS / 1000} seconds, Max: ${EXPECTED_MAX_MS / 1000} seconds`, - ); - - // Should return exit code 69 for timeout - expect(code).toEqual(EXIT_CODES.EX_UNAVAILABLE); - - // Should contain timeout-related message - expect(stdout).toContain('SNYK-CLI-0026'); - - // Should timeout within expected bounds (not wait for full server delay) - expect(duration).toBeGreaterThanOrEqual(EXPECTED_MIN_MS); - expect(duration).toBeLessThan(EXPECTED_MAX_MS); - - // Should send instrumentation data even on timeout - const requests = server.getRequests(); - const instrumentationRequest = requests.find((r) => - r.url?.includes(`/api/hidden/orgs/${orgId}/analytics`), - ); - expect(instrumentationRequest).toBeDefined(); - }, - ); -});