diff --git a/src/cli/commands/test/iac/local-execution/types.ts b/src/cli/commands/test/iac/local-execution/types.ts index 249e90db8d0..ab6b7fab219 100644 --- a/src/cli/commands/test/iac/local-execution/types.ts +++ b/src/cli/commands/test/iac/local-execution/types.ts @@ -353,6 +353,8 @@ export enum IaCErrorCodes { // Unified Policy Engine executable errors. InvalidUserPolicyEnginePathError = 1140, + FailedToDownloadPolicyEngineError = 1141, + FailedToCachePolicyEngineError = 1142, // Scan errors PolicyEngineScanError = 1150, diff --git a/src/lib/iac/file-utils.ts b/src/lib/iac/file-utils.ts index ec1b04a4442..f10b162ff73 100644 --- a/src/lib/iac/file-utils.ts +++ b/src/lib/iac/file-utils.ts @@ -44,3 +44,11 @@ export async function isArchive(path: string): Promise { return false; } } + +export async function saveFile( + dataBuffer: Buffer, + savePath: string, +): Promise { + await fsPromises.writeFile(savePath, dataBuffer); + await fsPromises.chmod(savePath, 0o744); +} diff --git a/src/lib/iac/test/v2/setup/local-cache/policy-engine/constants/index.ts b/src/lib/iac/test/v2/setup/local-cache/policy-engine/constants/index.ts index fc7cf10891c..8d3b2e71d80 100644 --- a/src/lib/iac/test/v2/setup/local-cache/policy-engine/constants/index.ts +++ b/src/lib/iac/test/v2/setup/local-cache/policy-engine/constants/index.ts @@ -3,9 +3,11 @@ import { formatPolicyEngineFileName } from './utils'; /** * The Policy Engine release version associated with this Snyk CLI version. */ -export const releaseVersion = '0.1.0'; +export const policyEngineReleaseVersion = '0.1.0'; /** * The Policy Engine executable's file name. */ -export const policyEngineFileName = formatPolicyEngineFileName(releaseVersion); +export const policyEngineFileName = formatPolicyEngineFileName( + policyEngineReleaseVersion, +); diff --git a/src/lib/iac/test/v2/setup/local-cache/policy-engine/download.ts b/src/lib/iac/test/v2/setup/local-cache/policy-engine/download.ts new file mode 100644 index 00000000000..fbc4fc50dad --- /dev/null +++ b/src/lib/iac/test/v2/setup/local-cache/policy-engine/download.ts @@ -0,0 +1,126 @@ +import * as pathLib from 'path'; +import * as crypto from 'crypto'; +import * as createDebugLogger from 'debug'; + +import { getErrorStringCode } from '../../../../../../../cli/commands/test/iac/local-execution/error-utils'; +import { IaCErrorCodes } from '../../../../../../../cli/commands/test/iac/local-execution/types'; +import { CustomError } from '../../../../../../errors'; +import { TimerMetricInstance } from '../../../../../../metrics'; +import { TestConfig } from '../../../types'; +import { fetchCacheResource } from '../utils'; +import { policyEngineFileName, policyEngineReleaseVersion } from './constants'; +import { saveFile } from '../../../../../file-utils'; + +const debugLog = createDebugLogger('snyk-iac'); + +export async function downloadPolicyEngine( + testConfig: TestConfig, +): Promise { + let downloadDurationSeconds = 0; + + const timer = new TimerMetricInstance('iac_policy_engine_download'); + timer.start(); + + const dataBuffer = await fetch(); + + assertValidChecksum(dataBuffer); + + const cachedPolicyEnginePath = await cache( + dataBuffer, + testConfig.iacCachePath, + ); + + timer.stop(); + downloadDurationSeconds = Math.round((timer.getValue() as number) / 1000); + + debugLog( + `Downladed Policy Engine successfully in ${downloadDurationSeconds} seconds`, + ); + + return cachedPolicyEnginePath; +} + +async function fetch(): Promise { + debugLog(`Fetching Policy Engine executable from ${policyEngineUrl}`); + + let policyEngineDataBuffer: Buffer; + try { + policyEngineDataBuffer = await fetchCacheResource(policyEngineUrl); + } catch (err) { + throw new FailedToDownloadPolicyEngineError(); + } + debugLog('Policy Engine executable was fetched successfully'); + + return policyEngineDataBuffer; +} + +export const policyEngineUrl = `https://static.snyk.io/cli/iac/test/v${policyEngineReleaseVersion}/${policyEngineFileName}`; + +export class FailedToDownloadPolicyEngineError extends CustomError { + constructor() { + super(`Failed to download cache resource from ${policyEngineUrl}`); + this.code = IaCErrorCodes.FailedToDownloadPolicyEngineError; + this.strCode = getErrorStringCode(this.code); + this.userMessage = + `Could not fetch cache resource from: ${policyEngineUrl}` + + '\nEnsure valid network connection.'; + } +} + +function assertValidChecksum(dataBuffer: Buffer): void { + const computedChecksum = crypto + .createHash('sha256') + .update(dataBuffer) + .digest('hex'); + + if (computedChecksum !== policyEngineChecksum) { + throw new FailedToDownloadPolicyEngineError(); + } + + debugLog('Fetched Policy Engine executable has valid checksum'); +} + +export const policyEngineChecksum = { + 'snyk-iac-test_0.1.0_Linux_x86_64': + '0b0d846cd74bf42676f79875ab30f20dc08529aedb94f2f6dd31a67c302b78e4', + 'snyk-iac-test_0.1.0_Darwin_x86_64': + '896960a09b6adf699185f443428cda25ccb30123440bc8735e1a05df1e9cbc12', + 'snyk-iac-test_0.1.0_Windows_x86_64.exe': + 'a0b0b5781218f42d121cd1c6bca2d5ea27fbe76b1e9817e37745586c276cabda', + 'snyk-iac-test_0.1.0_Linux_arm64': + 'b0c0bd9a06cc3d556b526b868e5a4ac4c5a3938899c4e312607f6828626debe9', + 'snyk-iac-test_0.1.0_Darwin_arm64': + 'cf7a327378983810ea043774a333bdd724e3866a815555d59ec5cd8aa25ea5ec', + 'snyk-iac-test_0.1.0_Windows_arm64.exe': + 'd88b24c611c8d37c2df9382e6e1b39933b926ee2ed438f1fdb69c51da5086fc5', +}[policyEngineFileName]!; + +async function cache( + dataBuffer: Buffer, + iacCachePath: string, +): Promise { + const savePath = pathLib.join(iacCachePath, policyEngineFileName); + + debugLog(`Caching Policy Engine executable to ${savePath}`); + + try { + await saveFile(dataBuffer, savePath); + } catch (err) { + throw new FailedToCachePolicyEngineError(savePath); + } + + debugLog(`Policy Engine executable was successfully cached`); + + return savePath; +} + +export class FailedToCachePolicyEngineError extends CustomError { + constructor(savePath: string) { + super(`Failed to cache Policy Engine executable to ${savePath}`); + this.code = IaCErrorCodes.FailedToCachePolicyEngineError; + this.strCode = getErrorStringCode(this.code); + this.userMessage = + `Could not write the downloaded cache resource to: ${savePath}` + + '\nEnsure the cache directory is writable.'; + } +} diff --git a/src/lib/iac/test/v2/setup/local-cache/policy-engine/index.ts b/src/lib/iac/test/v2/setup/local-cache/policy-engine/index.ts index 515027805a9..04b4958ca40 100644 --- a/src/lib/iac/test/v2/setup/local-cache/policy-engine/index.ts +++ b/src/lib/iac/test/v2/setup/local-cache/policy-engine/index.ts @@ -1,27 +1,21 @@ import * as createDebugLogger from 'debug'; -import { CustomError } from '../../../../../../errors'; import { TestConfig } from '../../../types'; import { lookupLocalPolicyEngine } from './lookup-local'; +import { downloadPolicyEngine } from './download'; const debugLogger = createDebugLogger('snyk-iac'); export async function initPolicyEngine(testConfig: TestConfig) { debugLogger('Looking for Policy Engine locally'); - const localPolicyEnginePath = await lookupLocalPolicyEngine(testConfig); + let policyEnginePath = await lookupLocalPolicyEngine(testConfig); - if (!localPolicyEnginePath) { + if (!policyEnginePath) { debugLogger( `Downloading the Policy Engine and saving it at ${testConfig.iacCachePath}`, ); - // TODO: Download Policy Engine executable + policyEnginePath = await downloadPolicyEngine(testConfig); } - if (localPolicyEnginePath) { - return localPolicyEnginePath; - } else { - throw new CustomError( - 'Could not find a valid Policy Engine in the configured path', - ); - } + return policyEnginePath; } diff --git a/src/lib/iac/test/v2/setup/local-cache/utils.ts b/src/lib/iac/test/v2/setup/local-cache/utils.ts index 3ffbc1fffad..f7428e5ed59 100644 --- a/src/lib/iac/test/v2/setup/local-cache/utils.ts +++ b/src/lib/iac/test/v2/setup/local-cache/utils.ts @@ -1,6 +1,8 @@ import * as createDebugLogger from 'debug'; import * as path from 'path'; + import { CustomError } from '../../../../../errors'; +import { makeRequest } from '../../../../../request'; const debugLogger = createDebugLogger('snyk-iac'); @@ -37,3 +39,15 @@ export class InvalidUserPathError extends CustomError { super(message); } } + +export async function fetchCacheResource(url: string): Promise { + const { res, body: cacheResourceBuffer } = await makeRequest({ + url, + }); + + if (res.statusCode !== 200) { + throw new CustomError(`Failed to download cache resource from ${url}`); + } + + return cacheResourceBuffer; +} diff --git a/test/jest/unit/lib/iac/test/v2/setup/local-cache/policy-engine/download.spec.ts b/test/jest/unit/lib/iac/test/v2/setup/local-cache/policy-engine/download.spec.ts new file mode 100644 index 00000000000..f9db94e2c73 --- /dev/null +++ b/test/jest/unit/lib/iac/test/v2/setup/local-cache/policy-engine/download.spec.ts @@ -0,0 +1,141 @@ +import * as crypto from 'crypto'; +import * as pathLib from 'path'; +import * as cloneDeep from 'lodash.clonedeep'; + +import * as localCacheUtils from '../../../../../../../../../../src/lib/iac/test/v2/setup/local-cache/utils'; +import * as fileUtils from '../../../../../../../../../../src/lib/iac/file-utils'; +import { TestConfig } from '../../../../../../../../../../src/lib/iac/test/v2/types'; +import { + downloadPolicyEngine, + FailedToCachePolicyEngineError, + FailedToDownloadPolicyEngineError, + policyEngineChecksum, + policyEngineUrl, +} from '../../../../../../../../../../src/lib/iac/test/v2/setup/local-cache/policy-engine/download'; + +jest.mock( + '../../../../../../../../../../src/lib/iac/test/v2/setup/local-cache/policy-engine/constants', + () => ({ + policyEngineFileName: 'test-policy-engine-file-name', + }), +); + +describe('downloadPolicyEngine', () => { + const testIacCachePath = pathLib.join('test', 'iac', 'cache', 'path'); + const defaultTestTestConfig = { + iacCachePath: testIacCachePath, + }; + const testPolicyEngineFileName = 'test-policy-engine-file-name'; + const testCachedPolicyEnginePath = pathLib.join( + testIacCachePath, + testPolicyEngineFileName, + ); + + const defaultHashMock = { + update: jest.fn().mockReturnThis(), + digest: jest.fn().mockReturnValue(policyEngineChecksum), + } as any; + + afterEach(() => { + jest.restoreAllMocks(); + }); + + it('fetches the Policy Engine executable', async () => { + // Arrange + const testTestConfig: TestConfig = cloneDeep(defaultTestTestConfig); + const testDataBuffer = Buffer.from('test-data-buffer'); + + const fetchCacheResourceSpy = jest + .spyOn(localCacheUtils, 'fetchCacheResource') + .mockResolvedValue(testDataBuffer); + jest.spyOn(crypto, 'createHash').mockReturnValue(defaultHashMock); + jest.spyOn(fileUtils, 'saveFile').mockResolvedValue(); + + // Act + await downloadPolicyEngine(testTestConfig); + + // Assert + expect(fetchCacheResourceSpy).toHaveBeenCalledWith(policyEngineUrl); + }); + + it('caches the fetched cache resource', async () => { + // Arrange + const testTestConfig: TestConfig = cloneDeep(defaultTestTestConfig); + const testDataBuffer = Buffer.from('test-data-buffer'); + + jest + .spyOn(localCacheUtils, 'fetchCacheResource') + .mockResolvedValue(testDataBuffer); + jest.spyOn(crypto, 'createHash').mockReturnValue(defaultHashMock); + const saveCacheResourceSpy = jest + .spyOn(fileUtils, 'saveFile') + .mockResolvedValue(); + + // Act + await downloadPolicyEngine(testTestConfig); + + // Assert + expect(saveCacheResourceSpy).toHaveBeenCalledWith( + testDataBuffer, + testCachedPolicyEnginePath, + ); + }); + + describe('when the Policy Engine executable fails to be fetched', () => { + it('throws an error', async () => { + // Arrange + const testTestConfig: TestConfig = cloneDeep(defaultTestTestConfig); + jest + .spyOn(localCacheUtils, 'fetchCacheResource') + .mockRejectedValue(new Error()); + jest.spyOn(crypto, 'createHash').mockReturnValue(defaultHashMock); + jest.spyOn(fileUtils, 'saveFile').mockResolvedValue(); + + // Act + Assert + await expect(downloadPolicyEngine(testTestConfig)).rejects.toThrow( + FailedToDownloadPolicyEngineError, + ); + }); + }); + + describe('when the Policy engine executable has an invalid checksum', () => { + it('throws an error', async () => { + // Arrange + const testTestConfig: TestConfig = cloneDeep(defaultTestTestConfig); + const testDataBuffer = Buffer.from('test-data-buffer'); + + jest + .spyOn(localCacheUtils, 'fetchCacheResource') + .mockResolvedValue(testDataBuffer); + jest.spyOn(crypto, 'createHash').mockReturnValue({ + ...defaultHashMock, + digest: () => 'test-inconsistent-checksum', + }); + jest.spyOn(fileUtils, 'saveFile').mockResolvedValue(); + + // Act + Assert + await expect(downloadPolicyEngine(testTestConfig)).rejects.toThrow( + FailedToDownloadPolicyEngineError, + ); + }); + }); + + describe('when the Policy Engine executable fails to be cached', () => { + it('throws an error', async () => { + // Arrange + const testTestConfig: TestConfig = cloneDeep(defaultTestTestConfig); + const testDataBuffer = Buffer.from('test-data-buffer'); + + jest + .spyOn(localCacheUtils, 'fetchCacheResource') + .mockResolvedValue(testDataBuffer); + jest.spyOn(crypto, 'createHash').mockReturnValue(defaultHashMock); + jest.spyOn(fileUtils, 'saveFile').mockRejectedValue(new Error()); + + // Act + Assert + await expect(downloadPolicyEngine(testTestConfig)).rejects.toThrow( + FailedToCachePolicyEngineError, + ); + }); + }); +}); diff --git a/test/jest/unit/lib/iac/test/v2/setup/local-cache/policy-engine/index.spec.ts b/test/jest/unit/lib/iac/test/v2/setup/local-cache/policy-engine/index.spec.ts new file mode 100644 index 00000000000..c0e6dca8e66 --- /dev/null +++ b/test/jest/unit/lib/iac/test/v2/setup/local-cache/policy-engine/index.spec.ts @@ -0,0 +1,123 @@ +import * as pathLib from 'path'; +import * as cloneDeep from 'lodash.clonedeep'; + +import * as lookupLocalLib from '../../../../../../../../../../src/lib/iac/test/v2/setup/local-cache/policy-engine/lookup-local'; +import * as downloadLib from '../../../../../../../../../../src/lib/iac/test/v2/setup/local-cache/policy-engine/download'; +import { initPolicyEngine } from '../../../../../../../../../../src/lib/iac/test/v2/setup/local-cache/policy-engine'; + +jest.mock( + '../../../../../../../../../../src/lib/iac/test/v2/setup/local-cache/policy-engine/constants', + () => ({ + policyEngineFileName: 'test-policy-engine-file-name', + }), +); + +describe('initPolicyEngine', () => { + const testIacCachePath = pathLib.join('test', 'iac', 'cache', 'path'); + const defaultTestTestConfig = { + iacCachePath: testIacCachePath, + }; + const testPolicyEngineFileName = 'test-policy-engine-file-name'; + const testCachedPolicyEnginePath = pathLib.join( + testIacCachePath, + testPolicyEngineFileName, + ); + + afterEach(() => { + jest.restoreAllMocks(); + }); + + it('looks up the Policy Engine executable locally', async () => { + // Arrange + const testTestConfig = cloneDeep(defaultTestTestConfig); + + const lookupLocalSpy = jest + .spyOn(lookupLocalLib, 'lookupLocalPolicyEngine') + .mockResolvedValue(testCachedPolicyEnginePath); + jest + .spyOn(downloadLib, 'downloadPolicyEngine') + .mockResolvedValue(testCachedPolicyEnginePath); + + // Act + await initPolicyEngine(testTestConfig); + + // Assert + expect(lookupLocalSpy).toHaveBeenCalledWith(testTestConfig); + }); + + describe('when a valid local Policy Engine executable is found', () => { + it('returns the local Policy Engine executable path', async () => { + // Arrange + const testTestConfig = cloneDeep(defaultTestTestConfig); + + jest + .spyOn(lookupLocalLib, 'lookupLocalPolicyEngine') + .mockResolvedValue(testCachedPolicyEnginePath); + jest + .spyOn(downloadLib, 'downloadPolicyEngine') + .mockResolvedValue(testCachedPolicyEnginePath); + + // Act + const res = await initPolicyEngine(testTestConfig); + + // Assert + expect(res).toEqual(testCachedPolicyEnginePath); + }); + + it('does not download the Policy Engine executable', async () => { + // Arrange + const testTestConfig = cloneDeep(defaultTestTestConfig); + + jest + .spyOn(lookupLocalLib, 'lookupLocalPolicyEngine') + .mockResolvedValue(testCachedPolicyEnginePath); + const downloadSpy = jest + .spyOn(downloadLib, 'downloadPolicyEngine') + .mockResolvedValue(testCachedPolicyEnginePath); + + // Act + await initPolicyEngine(testTestConfig); + + // Assert + expect(downloadSpy).not.toHaveBeenCalled(); + }); + }); + + describe('when no valid local Policy Engine executable is found', () => { + it('downloads the Policy Engine executable', async () => { + // Arrange + const testTestConfig = cloneDeep(defaultTestTestConfig); + + jest + .spyOn(lookupLocalLib, 'lookupLocalPolicyEngine') + .mockResolvedValue(undefined); + const downloadSpy = jest + .spyOn(downloadLib, 'downloadPolicyEngine') + .mockResolvedValue(testCachedPolicyEnginePath); + + // Act + await initPolicyEngine(testTestConfig); + + // Assert + expect(downloadSpy).toHaveBeenCalledWith(testTestConfig); + }); + + it('returns the path to the downloaded Policy Engine executable', async () => { + // Arrange + const testTestConfig = cloneDeep(defaultTestTestConfig); + + jest + .spyOn(lookupLocalLib, 'lookupLocalPolicyEngine') + .mockResolvedValue(undefined); + jest + .spyOn(downloadLib, 'downloadPolicyEngine') + .mockResolvedValue(testCachedPolicyEnginePath); + + // Act + const res = await initPolicyEngine(testTestConfig); + + // Assert + expect(res).toEqual(testCachedPolicyEnginePath); + }); + }); +}); diff --git a/test/jest/unit/lib/iac/test/v2/setup/local-cache/utils.spec.ts b/test/jest/unit/lib/iac/test/v2/setup/local-cache/utils.spec.ts index f3b73785bef..d9d04c4b3e5 100644 --- a/test/jest/unit/lib/iac/test/v2/setup/local-cache/utils.spec.ts +++ b/test/jest/unit/lib/iac/test/v2/setup/local-cache/utils.spec.ts @@ -1,9 +1,13 @@ import * as pathLib from 'path'; import * as cloneDeep from 'lodash.clonedeep'; + +import * as requestLib from '../../../../../../../../../src/lib/request'; import { + fetchCacheResource, InvalidUserPathError, lookupLocal, } from '../../../../../../../../../src/lib/iac/test/v2/setup/local-cache/utils'; +import { CustomError } from '../../../../../../../../../src/lib/errors'; describe('lookupLocal', () => { const iacCachePath = pathLib.join('iac', 'cache', 'path'); @@ -99,3 +103,65 @@ describe('lookupLocal', () => { }); }); }); + +describe('fetchCacheResource', () => { + afterEach(() => { + jest.restoreAllMocks(); + }); + + it('fetches the cache resource', async () => { + // Arrange + const testCacheResourceUrl = 'test-cache-resource-url'; + const testCacheResourcBuffer = Buffer.from('test-cache-resource-content'); + + const makeRequestSpy = jest + .spyOn(requestLib, 'makeRequest') + .mockResolvedValue({ + body: testCacheResourcBuffer, + res: { + statusCode: 200, + body: testCacheResourcBuffer, + } as any, + } as any); + + // Act + await fetchCacheResource(testCacheResourceUrl); + + // Assert + expect(makeRequestSpy).toHaveBeenCalledWith({ + url: testCacheResourceUrl, + }); + }); + + describe('when the request fails to be sent', () => { + it('throws an error', async () => { + // Arrange + const testCacheResourceUrl = 'test-cache-resource-url'; + + jest.spyOn(requestLib, 'makeRequest').mockRejectedValue(new Error()); + + // Act + Assert + await expect( + fetchCacheResource(testCacheResourceUrl), + ).rejects.toThrowError(); + }); + }); + describe('when an error response is received', () => { + it('throws an error', async () => { + // Arrange + const testCacheResourceUrl = 'test-cache-resource-url'; + + jest.spyOn(requestLib, 'makeRequest').mockResolvedValue({ + body: '', + res: { + statusCode: 500, + }, + } as any); + + // Act + Assert + await expect(fetchCacheResource(testCacheResourceUrl)).rejects.toThrow( + CustomError, + ); + }); + }); +});