Merge pull request #3327 from snyk/chore/download-upe-cfg-1885

chore: Implement UPE executable downloading

src/cli/commands/test/iac/local-execution/types.ts

@@ -353,6 +353,8 @@ export enum IaCErrorCodes {
 
   // Unified Policy Engine executable errors.
   InvalidUserPolicyEnginePathError = 1140,
+  FailedToDownloadPolicyEngineError = 1141,
+  FailedToCachePolicyEngineError = 1142,
 
   // Scan errors
   PolicyEngineScanError = 1150,

src/lib/iac/file-utils.ts

@@ -44,3 +44,11 @@ export async function isArchive(path: string): Promise<boolean> {
     return false;
   }
 }
+
+export async function saveFile(
+  dataBuffer: Buffer,
+  savePath: string,
+): Promise<void> {
+  await fsPromises.writeFile(savePath, dataBuffer);
+  await fsPromises.chmod(savePath, 0o744);
+}

src/lib/iac/test/v2/setup/local-cache/policy-engine/constants/index.ts

@@ -3,9 +3,11 @@ import { formatPolicyEngineFileName } from './utils';
 /**
  * The Policy Engine release version associated with this Snyk CLI version.
  */
-export const releaseVersion = '0.1.0';
+export const policyEngineReleaseVersion = '0.1.0';
 
 /**
  * The Policy Engine executable's file name.
  */
-export const policyEngineFileName = formatPolicyEngineFileName(releaseVersion);
+export const policyEngineFileName = formatPolicyEngineFileName(
+  policyEngineReleaseVersion,
+);

src/lib/iac/test/v2/setup/local-cache/policy-engine/download.ts

@@ -0,0 +1,126 @@
+import * as pathLib from 'path';
+import * as crypto from 'crypto';
+import * as createDebugLogger from 'debug';
+
+import { getErrorStringCode } from '../../../../../../../cli/commands/test/iac/local-execution/error-utils';
+import { IaCErrorCodes } from '../../../../../../../cli/commands/test/iac/local-execution/types';
+import { CustomError } from '../../../../../../errors';
+import { TimerMetricInstance } from '../../../../../../metrics';
+import { TestConfig } from '../../../types';
+import { fetchCacheResource } from '../utils';
+import { policyEngineFileName, policyEngineReleaseVersion } from './constants';
+import { saveFile } from '../../../../../file-utils';
+
+const debugLog = createDebugLogger('snyk-iac');
+
+export async function downloadPolicyEngine(
+  testConfig: TestConfig,
+): Promise<string> {
+  let downloadDurationSeconds = 0;
+
+  const timer = new TimerMetricInstance('iac_policy_engine_download');
+  timer.start();
+
+  const dataBuffer = await fetch();
+
+  assertValidChecksum(dataBuffer);
+
+  const cachedPolicyEnginePath = await cache(
+    dataBuffer,
+    testConfig.iacCachePath,
+  );
+
+  timer.stop();
+  downloadDurationSeconds = Math.round((timer.getValue() as number) / 1000);
+
+  debugLog(
+    `Downladed Policy Engine successfully in ${downloadDurationSeconds} seconds`,
+  );
+
+  return cachedPolicyEnginePath;
+}
+
+async function fetch(): Promise<Buffer> {
+  debugLog(`Fetching Policy Engine executable from ${policyEngineUrl}`);
+
+  let policyEngineDataBuffer: Buffer;
+  try {
+    policyEngineDataBuffer = await fetchCacheResource(policyEngineUrl);
+  } catch (err) {
+    throw new FailedToDownloadPolicyEngineError();
+  }
+  debugLog('Policy Engine executable was fetched successfully');
+
+  return policyEngineDataBuffer;
+}
+
+export const policyEngineUrl = `https://static.snyk.io/cli/iac/test/v${policyEngineReleaseVersion}/${policyEngineFileName}`;
+
+export class FailedToDownloadPolicyEngineError extends CustomError {
+  constructor() {
+    super(`Failed to download cache resource from ${policyEngineUrl}`);
+    this.code = IaCErrorCodes.FailedToDownloadPolicyEngineError;
+    this.strCode = getErrorStringCode(this.code);
+    this.userMessage =
+      `Could not fetch cache resource from: ${policyEngineUrl}` +
+      '\nEnsure valid network connection.';
+  }
+}
+
+function assertValidChecksum(dataBuffer: Buffer): void {
+  const computedChecksum = crypto
+    .createHash('sha256')
+    .update(dataBuffer)
+    .digest('hex');
+
+  if (computedChecksum !== policyEngineChecksum) {
+    throw new FailedToDownloadPolicyEngineError();
+  }
+
+  debugLog('Fetched Policy Engine executable has valid checksum');
+}
+
+export const policyEngineChecksum = {
+  'snyk-iac-test_0.1.0_Linux_x86_64':
+    '0b0d846cd74bf42676f79875ab30f20dc08529aedb94f2f6dd31a67c302b78e4',
+  'snyk-iac-test_0.1.0_Darwin_x86_64':
+    '896960a09b6adf699185f443428cda25ccb30123440bc8735e1a05df1e9cbc12',
+  'snyk-iac-test_0.1.0_Windows_x86_64.exe':
+    'a0b0b5781218f42d121cd1c6bca2d5ea27fbe76b1e9817e37745586c276cabda',
+  'snyk-iac-test_0.1.0_Linux_arm64':
+    'b0c0bd9a06cc3d556b526b868e5a4ac4c5a3938899c4e312607f6828626debe9',
+  'snyk-iac-test_0.1.0_Darwin_arm64':
+    'cf7a327378983810ea043774a333bdd724e3866a815555d59ec5cd8aa25ea5ec',
+  'snyk-iac-test_0.1.0_Windows_arm64.exe':
+    'd88b24c611c8d37c2df9382e6e1b39933b926ee2ed438f1fdb69c51da5086fc5',
+}[policyEngineFileName]!;
+
+async function cache(
+  dataBuffer: Buffer,
+  iacCachePath: string,
+): Promise<string> {
+  const savePath = pathLib.join(iacCachePath, policyEngineFileName);
+
+  debugLog(`Caching Policy Engine executable to ${savePath}`);
+
+  try {
+    await saveFile(dataBuffer, savePath);
+  } catch (err) {
+    throw new FailedToCachePolicyEngineError(savePath);
+  }
+
+  debugLog(`Policy Engine executable was successfully cached`);
+
+  return savePath;
+}
+
+export class FailedToCachePolicyEngineError extends CustomError {
+  constructor(savePath: string) {
+    super(`Failed to cache Policy Engine executable to ${savePath}`);
+    this.code = IaCErrorCodes.FailedToCachePolicyEngineError;
+    this.strCode = getErrorStringCode(this.code);
+    this.userMessage =
+      `Could not write the downloaded cache resource to: ${savePath}` +
+      '\nEnsure the cache directory is writable.';
+  }
+}

src/lib/iac/test/v2/setup/local-cache/policy-engine/index.ts

@@ -1,27 +1,21 @@
 import * as createDebugLogger from 'debug';
 
-import { CustomError } from '../../../../../../errors';
 import { TestConfig } from '../../../types';
 import { lookupLocalPolicyEngine } from './lookup-local';
+import { downloadPolicyEngine } from './download';
 
 const debugLogger = createDebugLogger('snyk-iac');
 
 export async function initPolicyEngine(testConfig: TestConfig) {
   debugLogger('Looking for Policy Engine locally');
-  const localPolicyEnginePath = await lookupLocalPolicyEngine(testConfig);
+  let policyEnginePath = await lookupLocalPolicyEngine(testConfig);
 
-  if (!localPolicyEnginePath) {
+  if (!policyEnginePath) {
     debugLogger(
       `Downloading the Policy Engine and saving it at ${testConfig.iacCachePath}`,
     );
-    // TODO: Download Policy Engine executable
+    policyEnginePath = await downloadPolicyEngine(testConfig);
   }
 
-  if (localPolicyEnginePath) {
-    return localPolicyEnginePath;
-  } else {
-    throw new CustomError(
-      'Could not find a valid Policy Engine in the configured path',
-    );
-  }
+  return policyEnginePath;
 }

src/lib/iac/test/v2/setup/local-cache/utils.ts

@@ -1,6 +1,8 @@
 import * as createDebugLogger from 'debug';
 import * as path from 'path';
+
 import { CustomError } from '../../../../../errors';
+import { makeRequest } from '../../../../../request';
 
 const debugLogger = createDebugLogger('snyk-iac');
 
@@ -37,3 +39,15 @@ export class InvalidUserPathError extends CustomError {
     super(message);
   }
 }
+
+export async function fetchCacheResource(url: string): Promise<Buffer> {
+  const { res, body: cacheResourceBuffer } = await makeRequest({
+    url,
+  });
+
+  if (res.statusCode !== 200) {
+    throw new CustomError(`Failed to download cache resource from ${url}`);
+  }
+
+  return cacheResourceBuffer;
+}

test/jest/unit/lib/iac/test/v2/setup/local-cache/policy-engine/download.spec.ts

@@ -0,0 +1,141 @@
+import * as crypto from 'crypto';
+import * as pathLib from 'path';
+import * as cloneDeep from 'lodash.clonedeep';
+
+import * as localCacheUtils from '../../../../../../../../../../src/lib/iac/test/v2/setup/local-cache/utils';
+import * as fileUtils from '../../../../../../../../../../src/lib/iac/file-utils';
+import { TestConfig } from '../../../../../../../../../../src/lib/iac/test/v2/types';
+import {
+  downloadPolicyEngine,
+  FailedToCachePolicyEngineError,
+  FailedToDownloadPolicyEngineError,
+  policyEngineChecksum,
+  policyEngineUrl,
+} from '../../../../../../../../../../src/lib/iac/test/v2/setup/local-cache/policy-engine/download';
+
+jest.mock(
+  '../../../../../../../../../../src/lib/iac/test/v2/setup/local-cache/policy-engine/constants',
+  () => ({
+    policyEngineFileName: 'test-policy-engine-file-name',
+  }),
+);
+
+describe('downloadPolicyEngine', () => {
+  const testIacCachePath = pathLib.join('test', 'iac', 'cache', 'path');
+  const defaultTestTestConfig = {
+    iacCachePath: testIacCachePath,
+  };
+  const testPolicyEngineFileName = 'test-policy-engine-file-name';
+  const testCachedPolicyEnginePath = pathLib.join(
+    testIacCachePath,
+    testPolicyEngineFileName,
+  );
+
+  const defaultHashMock = {
+    update: jest.fn().mockReturnThis(),
+    digest: jest.fn().mockReturnValue(policyEngineChecksum),
+  } as any;
+
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  it('fetches the Policy Engine executable', async () => {
+    // Arrange
+    const testTestConfig: TestConfig = cloneDeep(defaultTestTestConfig);
+    const testDataBuffer = Buffer.from('test-data-buffer');
+
+    const fetchCacheResourceSpy = jest
+      .spyOn(localCacheUtils, 'fetchCacheResource')
+      .mockResolvedValue(testDataBuffer);
+    jest.spyOn(crypto, 'createHash').mockReturnValue(defaultHashMock);
+    jest.spyOn(fileUtils, 'saveFile').mockResolvedValue();
+
+    // Act
+    await downloadPolicyEngine(testTestConfig);
+
+    // Assert
+    expect(fetchCacheResourceSpy).toHaveBeenCalledWith(policyEngineUrl);
+  });
+
+  it('caches the fetched cache resource', async () => {
+    // Arrange
+    const testTestConfig: TestConfig = cloneDeep(defaultTestTestConfig);
+    const testDataBuffer = Buffer.from('test-data-buffer');
+
+    jest
+      .spyOn(localCacheUtils, 'fetchCacheResource')
+      .mockResolvedValue(testDataBuffer);
+    jest.spyOn(crypto, 'createHash').mockReturnValue(defaultHashMock);
+    const saveCacheResourceSpy = jest
+      .spyOn(fileUtils, 'saveFile')
+      .mockResolvedValue();
+
+    // Act
+    await downloadPolicyEngine(testTestConfig);
+
+    // Assert
+    expect(saveCacheResourceSpy).toHaveBeenCalledWith(
+      testDataBuffer,
+      testCachedPolicyEnginePath,
+    );
+  });
+
+  describe('when the Policy Engine executable fails to be fetched', () => {
+    it('throws an error', async () => {
+      // Arrange
+      const testTestConfig: TestConfig = cloneDeep(defaultTestTestConfig);
+      jest
+        .spyOn(localCacheUtils, 'fetchCacheResource')
+        .mockRejectedValue(new Error());
+      jest.spyOn(crypto, 'createHash').mockReturnValue(defaultHashMock);
+      jest.spyOn(fileUtils, 'saveFile').mockResolvedValue();
+
+      // Act + Assert
+      await expect(downloadPolicyEngine(testTestConfig)).rejects.toThrow(
+        FailedToDownloadPolicyEngineError,
+      );
+    });
+  });
+
+  describe('when the Policy engine executable has an invalid checksum', () => {
+    it('throws an error', async () => {
+      // Arrange
+      const testTestConfig: TestConfig = cloneDeep(defaultTestTestConfig);
+      const testDataBuffer = Buffer.from('test-data-buffer');
+
+      jest
+        .spyOn(localCacheUtils, 'fetchCacheResource')
+        .mockResolvedValue(testDataBuffer);
+      jest.spyOn(crypto, 'createHash').mockReturnValue({
+        ...defaultHashMock,
+        digest: () => 'test-inconsistent-checksum',
+      });
+      jest.spyOn(fileUtils, 'saveFile').mockResolvedValue();
+
+      // Act + Assert
+      await expect(downloadPolicyEngine(testTestConfig)).rejects.toThrow(
+        FailedToDownloadPolicyEngineError,
+      );
+    });
+  });
+
+  describe('when the Policy Engine executable fails to be cached', () => {
+    it('throws an error', async () => {
+      // Arrange
+      const testTestConfig: TestConfig = cloneDeep(defaultTestTestConfig);
+      const testDataBuffer = Buffer.from('test-data-buffer');
+
+      jest
+        .spyOn(localCacheUtils, 'fetchCacheResource')
+        .mockResolvedValue(testDataBuffer);
+      jest.spyOn(crypto, 'createHash').mockReturnValue(defaultHashMock);
+      jest.spyOn(fileUtils, 'saveFile').mockRejectedValue(new Error());
+
+      // Act + Assert
+      await expect(downloadPolicyEngine(testTestConfig)).rejects.toThrow(
+        FailedToCachePolicyEngineError,
+      );
+    });
+  });
+});