Skip to content

Commit

Permalink
fix: return exit code 3 when no resources can be found
Browse files Browse the repository at this point in the history
  • Loading branch information
@ofekatr
ofekatr committed Aug 15, 2022
1 parent 282b025 commit 9d2e41f
Show file tree
Hide file tree
Showing 4 changed files with 240 additions and 43 deletions.
60 changes: 47 additions & 13 deletions src/lib/iac/test/v2/output.ts
View file
Expand Up @@ -13,12 +13,16 @@ import {
} from '../../../formatters/iac-output';
import { formatSnykIacTestTestData } from '../../../formatters/iac-output';
import { jsonStringifyLargeObject } from '../../../json';
import { IacOrgSettings } from '../../../../cli/commands/test/iac/local-execution/types';
import {
IaCErrorCodes,
IacOrgSettings,
} from '../../../../cli/commands/test/iac/local-execution/types';
import { convertEngineToSarifResults } from './sarif';
import { CustomError, FormattedCustomError } from '../../../errors';
import { SnykIacTestError } from './errors';
import stripAnsi from 'strip-ansi';
import * as path from 'path';
import { getErrorStringCode } from '../../../../cli/commands/test/iac/local-execution/error-utils';

export function buildOutput({
scanResult,
Expand Down Expand Up @@ -82,15 +86,11 @@ function buildTestCommandResultData({
convertEngineToSarifResults(scanResult),
);

const isPartialSuccess =
scanResult.results?.resources?.length || !scanResult.errors?.length;
if (!isPartialSuccess) {
throw new NoSuccessfulScansError(
{ json: jsonData, sarif: sarifData },
scanResult.errors!,
options,
);
}
assertHasSuccessfulScans(
scanResult,
{ json: jsonData, sarif: sarifData },
options,
);

let responseData: string;
if (options.json) {
Expand All @@ -101,8 +101,8 @@ function buildTestCommandResultData({
responseData = buildTextOutput({ scanResult, projectName, orgSettings });
}

const isFoundIssues = !!scanResult.results?.vulnerabilities?.length;
if (isFoundIssues) {
const hasVulnerabilities = !!scanResult.results?.vulnerabilities?.length;
if (hasVulnerabilities) {
throw new FoundIssuesError({
response: responseData,
json: jsonData,
Expand Down Expand Up @@ -156,6 +156,26 @@ function buildTextOutput({
return response;
}

function assertHasSuccessfulScans(
scanResult: TestOutput,
responseData: Omit<ResponseData, 'response'>,
options: { json?: boolean; sarif?: boolean },
): void {
const hasResources = !!scanResult.results?.resources?.length;
const hasErrors = !!scanResult.errors?.length;
const hasSuccessfulScans = hasResources || !hasErrors;

if (!hasSuccessfulScans) {
const hasLoadableInput = scanResult.errors!.some(
(error) => error.code !== IaCErrorCodes.NoLoadableInput,
);

throw hasLoadableInput
? new NoSuccessfulScansError(responseData, scanResult.errors!, options)
: new NoLoadableInputError(responseData, scanResult.errors!, options);
}
}

interface ResponseData {
response: string;
json: string;
Expand Down Expand Up @@ -191,8 +211,9 @@ export class NoSuccessfulScansError extends FormattedCustomError {
)
: stripAnsi(message),
);
this.strCode = firstErr.strCode;

this.code = firstErr.code;
this.strCode = firstErr.strCode;
this.json = isText ? responseData.json : message;
this.jsonStringifiedResults = responseData.json;
this.sarifStringifiedResults = responseData.sarif;
Expand All @@ -208,6 +229,19 @@ export class NoSuccessfulScansError extends FormattedCustomError {
}
}

export class NoLoadableInputError extends NoSuccessfulScansError {
constructor(
responseData: Omit<ResponseData, 'response'>,
errors: SnykIacTestError[],
options: { json?: boolean; sarif?: boolean },
) {
super(responseData, errors, options);

(this.code = IaCErrorCodes.NoFilesToScanError),
(this.strCode = getErrorStringCode(this.code));
}
}

export class FoundIssuesError extends CustomError {
public jsonStringifiedResults: string;
public sarifStringifiedResults: string;
Expand Down
209 changes: 186 additions & 23 deletions test/jest/unit/cli/commands/test/iac/v2/index.spec.ts
View file
Expand Up @@ -13,6 +13,7 @@ import { IacOrgSettings } from '../../../../../../../../src/cli/commands/test/ia
import { SnykIacTestError } from '../../../../../../../../src/lib/iac/test/v2/errors';
import {
FoundIssuesError,
NoLoadableInputError,
NoSuccessfulScansError,
} from '../../../../../../../../src/lib/iac/test/v2/output';

Expand Down Expand Up @@ -68,6 +69,18 @@ describe('test', () => {
errors: scanFixture.errors,
};

const scanWithoutLoadableInputsFixture = {
errors: [
new SnykIacTestError({
code: 2114,
message: 'no loadable input: path/to/test',
fields: {
path: 'path/to/test',
},
}),
],
};

beforeEach(() => {
jest.spyOn(scanLib, 'scan').mockReturnValue(scanFixture);

Expand Down Expand Up @@ -133,27 +146,73 @@ describe('test', () => {
expect.objectContaining({
name: 'NoSuccessfulScansError',
message:
'invalid input for input type: /Users/yairzohar/snyk/upe-test/README.txt',
code: 2106,
strCode: 'INVALID_INPUT',
'failed to parse input: /Users/yairzohar/snyk/upe-test/invalid-cfn.yml',
code: 2105,
strCode: 'FAILED_TO_PARSE_INPUT',
fields: {
path: '/Users/yairzohar/snyk/upe-test/README.txt',
path: '/Users/yairzohar/snyk/upe-test/invalid-cfn.yml',
},
path: '/Users/yairzohar/snyk/upe-test/README.txt',
path: '/Users/yairzohar/snyk/upe-test/invalid-cfn.yml',
userMessage:
'Test Failures\n\n Invalid input\n Path: /Users/yairzohar/snyk/upe-test/README.txt',
'Test Failures\n\n Failed to parse input\n Path: /Users/yairzohar/snyk/upe-test/invalid-cfn.yml',
formattedUserMessage:
'Test Failures\n\n Invalid input\n Path: /Users/yairzohar/snyk/upe-test/README.txt',
'Test Failures\n\n Failed to parse input\n Path: /Users/yairzohar/snyk/upe-test/invalid-cfn.yml',
sarifStringifiedResults: expect.stringContaining(
`"$schema": "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json"`,
),
jsonStringifiedResults:
'[\n {\n "ok": false,\n "error": "invalid input for input type: /Users/yairzohar/snyk/upe-test/README.txt",\n "path": "/Users/yairzohar/snyk/upe-test/README.txt"\n }\n]',
'[\n {\n "ok": false,\n "error": "failed to parse input: /Users/yairzohar/snyk/upe-test/invalid-cfn.yml",\n "path": "/Users/yairzohar/snyk/upe-test/invalid-cfn.yml"\n }\n]',
json:
'[\n {\n "ok": false,\n "error": "invalid input for input type: /Users/yairzohar/snyk/upe-test/README.txt",\n "path": "/Users/yairzohar/snyk/upe-test/README.txt"\n }\n]',
'[\n {\n "ok": false,\n "error": "failed to parse input: /Users/yairzohar/snyk/upe-test/invalid-cfn.yml",\n "path": "/Users/yairzohar/snyk/upe-test/invalid-cfn.yml"\n }\n]',
}),
);
});

describe('without loadable inputs', () => {
beforeEach(() => {
jest
.spyOn(scanLib, 'scan')
.mockReturnValue(scanWithoutLoadableInputsFixture);
});

it('throws the expected error', async () => {
// Arrange
let error;

// Act
try {
await test(['path/to/test'], defaultOptions);
} catch (err) {
error = err;
}

// Assert
expect(error).toBeInstanceOf(NoLoadableInputError);
expect(error).toEqual(
expect.objectContaining({
name: 'NoLoadableInputError',
message: 'no loadable input: path/to/test',
code: 1010,
strCode: 'NO_FILES_TO_SCAN_ERROR',
fields: {
path: 'path/to/test',
},
path: 'path/to/test',
userMessage:
"Test Failures\n\n The Snyk CLI couldn't find any valid IaC configuration files to scan\n Path: path/to/test",
formattedUserMessage:
"Test Failures\n\n The Snyk CLI couldn't find any valid IaC configuration files to scan\n Path: path/to/test",
sarifStringifiedResults: expect.stringContaining(
`"$schema": "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json"`,
),
jsonStringifiedResults:
'[\n {\n "ok": false,\n "error": "no loadable input: path/to/test",\n "path": "path/to/test"\n }\n]',
json:
'[\n {\n "ok": false,\n "error": "no loadable input: path/to/test",\n "path": "path/to/test"\n }\n]',
}),
);
});
});
});

describe('with issues', () => {
Expand Down Expand Up @@ -207,27 +266,77 @@ describe('test', () => {
expect.objectContaining({
name: 'NoSuccessfulScansError',
message:
'[\n {\n "ok": false,\n "error": "invalid input for input type: /Users/yairzohar/snyk/upe-test/README.txt",\n "path": "/Users/yairzohar/snyk/upe-test/README.txt"\n }\n]',
code: 2106,
strCode: 'INVALID_INPUT',
'[\n {\n "ok": false,\n "error": "failed to parse input: /Users/yairzohar/snyk/upe-test/invalid-cfn.yml",\n "path": "/Users/yairzohar/snyk/upe-test/invalid-cfn.yml"\n }\n]',
code: 2105,
strCode: 'FAILED_TO_PARSE_INPUT',
fields: {
path: '/Users/yairzohar/snyk/upe-test/README.txt',
path: '/Users/yairzohar/snyk/upe-test/invalid-cfn.yml',
},
path: '/Users/yairzohar/snyk/upe-test/README.txt',
path: '/Users/yairzohar/snyk/upe-test/invalid-cfn.yml',
userMessage:
'[\n {\n "ok": false,\n "error": "invalid input for input type: /Users/yairzohar/snyk/upe-test/README.txt",\n "path": "/Users/yairzohar/snyk/upe-test/README.txt"\n }\n]',
'[\n {\n "ok": false,\n "error": "failed to parse input: /Users/yairzohar/snyk/upe-test/invalid-cfn.yml",\n "path": "/Users/yairzohar/snyk/upe-test/invalid-cfn.yml"\n }\n]',
formattedUserMessage:
'[\n {\n "ok": false,\n "error": "invalid input for input type: /Users/yairzohar/snyk/upe-test/README.txt",\n "path": "/Users/yairzohar/snyk/upe-test/README.txt"\n }\n]',
'[\n {\n "ok": false,\n "error": "failed to parse input: /Users/yairzohar/snyk/upe-test/invalid-cfn.yml",\n "path": "/Users/yairzohar/snyk/upe-test/invalid-cfn.yml"\n }\n]',
sarifStringifiedResults: expect.stringContaining(
`"$schema": "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json"`,
),
jsonStringifiedResults:
'[\n {\n "ok": false,\n "error": "invalid input for input type: /Users/yairzohar/snyk/upe-test/README.txt",\n "path": "/Users/yairzohar/snyk/upe-test/README.txt"\n }\n]',
'[\n {\n "ok": false,\n "error": "failed to parse input: /Users/yairzohar/snyk/upe-test/invalid-cfn.yml",\n "path": "/Users/yairzohar/snyk/upe-test/invalid-cfn.yml"\n }\n]',
json:
'[\n {\n "ok": false,\n "error": "invalid input for input type: /Users/yairzohar/snyk/upe-test/README.txt",\n "path": "/Users/yairzohar/snyk/upe-test/README.txt"\n }\n]',
'[\n {\n "ok": false,\n "error": "failed to parse input: /Users/yairzohar/snyk/upe-test/invalid-cfn.yml",\n "path": "/Users/yairzohar/snyk/upe-test/invalid-cfn.yml"\n }\n]',
}),
);
});

describe('without loadable inputs', () => {
beforeEach(() => {
jest
.spyOn(scanLib, 'scan')
.mockReturnValue(scanWithoutLoadableInputsFixture);
});

it('throws the expected error', async () => {
// Arrange
let error;

// Act
try {
await test(['path/to/test'], {
...defaultOptions,
json: true,
});
} catch (err) {
error = err;
}

// Assert
expect(error).toBeInstanceOf(NoLoadableInputError);
expect(error).toEqual(
expect.objectContaining({
name: 'NoLoadableInputError',
message:
'[\n {\n "ok": false,\n "error": "no loadable input: path/to/test",\n "path": "path/to/test"\n }\n]',
code: 1010,
strCode: 'NO_FILES_TO_SCAN_ERROR',
fields: {
path: 'path/to/test',
},
path: 'path/to/test',
userMessage:
'[\n {\n "ok": false,\n "error": "no loadable input: path/to/test",\n "path": "path/to/test"\n }\n]',
formattedUserMessage:
'[\n {\n "ok": false,\n "error": "no loadable input: path/to/test",\n "path": "path/to/test"\n }\n]',
sarifStringifiedResults: expect.stringContaining(
`"$schema": "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json"`,
),
jsonStringifiedResults:
'[\n {\n "ok": false,\n "error": "no loadable input: path/to/test",\n "path": "path/to/test"\n }\n]',
json:
'[\n {\n "ok": false,\n "error": "no loadable input: path/to/test",\n "path": "path/to/test"\n }\n]',
}),
);
});
});
});
});

Expand Down Expand Up @@ -277,12 +386,12 @@ describe('test', () => {
message: expect.stringContaining(
`"$schema": "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json"`,
),
code: 2106,
strCode: 'INVALID_INPUT',
code: 2105,
strCode: 'FAILED_TO_PARSE_INPUT',
fields: {
path: '/Users/yairzohar/snyk/upe-test/README.txt',
path: '/Users/yairzohar/snyk/upe-test/invalid-cfn.yml',
},
path: '/Users/yairzohar/snyk/upe-test/README.txt',
path: '/Users/yairzohar/snyk/upe-test/invalid-cfn.yml',
userMessage: expect.stringContaining(
`"$schema": "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json"`,
),
Expand All @@ -293,13 +402,67 @@ describe('test', () => {
`"$schema": "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json"`,
),
jsonStringifiedResults:
'[\n {\n "ok": false,\n "error": "invalid input for input type: /Users/yairzohar/snyk/upe-test/README.txt",\n "path": "/Users/yairzohar/snyk/upe-test/README.txt"\n }\n]',
'[\n {\n "ok": false,\n "error": "failed to parse input: /Users/yairzohar/snyk/upe-test/invalid-cfn.yml",\n "path": "/Users/yairzohar/snyk/upe-test/invalid-cfn.yml"\n }\n]',
json: expect.stringContaining(
`"$schema": "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json"`,
),
}),
);
});

describe('without loadable inputs', () => {
beforeEach(() => {
jest
.spyOn(scanLib, 'scan')
.mockReturnValue(scanWithoutLoadableInputsFixture);
});

it('throws the expected error', async () => {
// Arrange
let error;

// Act
try {
await test(['path/to/test'], {
...defaultOptions,
sarif: true,
});
} catch (err) {
error = err;
}

// Assert
expect(error).toBeInstanceOf(NoLoadableInputError);
expect(error).toEqual(
expect.objectContaining({
name: 'NoLoadableInputError',
message: expect.stringContaining(
`"$schema": "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json"`,
),
code: 1010,
strCode: 'NO_FILES_TO_SCAN_ERROR',
fields: {
path: 'path/to/test',
},
path: 'path/to/test',
userMessage: expect.stringContaining(
`"$schema": "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json"`,
),
formattedUserMessage: expect.stringContaining(
`"$schema": "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json"`,
),
sarifStringifiedResults: expect.stringContaining(
`"$schema": "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json"`,
),
jsonStringifiedResults:
'[\n {\n "ok": false,\n "error": "no loadable input: path/to/test",\n "path": "path/to/test"\n }\n]',
json: expect.stringContaining(
`"$schema": "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json"`,
),
}),
);
});
});
});
});
});

0 comments on commit 9d2e41f

Please sign in to comment.