feat: iac experimental directory support

src/cli/commands/test/iac-local-execution/file-loader.ts

@@ -0,0 +1,54 @@
+import { makeDirectoryIterator } from '../../../../lib/iac/makeDirectoryIterator';
+import * as fs from 'fs';
+import * as util from 'util';
+import { IacFileData, VALID_FILE_TYPES } from './types';
+import { getFileType } from '../../../../lib/iac/iac-parser';
+import { IacFileTypes } from '../../../../lib/iac/constants';
+import { isLocalFolder } from '../../../../lib/detect';
+
+const loadFileContents = util.promisify(fs.readFile);
+const DEFAULT_ENCODING = 'utf-8';
+
+export async function loadFiles(pathToScan): Promise<IacFileData[]> {
+  let filePaths = [pathToScan];
+  if (isLocalFolder(pathToScan)) {
+    filePaths = await getFilePathsFromDirectory(pathToScan);
+  }
+
+  const filesToScan: IacFileData[] = [];
+  for (const filePath of filePaths) {
+    const fileData = await tryLoadFileData(filePath);
+    if (fileData) filesToScan.push(fileData!);
+  }
+
+  if (filesToScan.length === 0) {
+    throw Error("Couldn't find valid IaC files");
+  }
+
+  return filesToScan;
+}
+
+function getFilePathsFromDirectory(pathToScan: string): Array<string> {
+  const directoryPaths = makeDirectoryIterator(pathToScan);
+
+  const directoryFilePaths: string[] = [];
+  for (const filePath of directoryPaths) {
+    directoryFilePaths.push(filePath);
+  }
+  return directoryFilePaths;
+}
+
+async function tryLoadFileData(
+  pathToScan: string,
+): Promise<IacFileData | null> {
+  const fileType = getFileType(pathToScan);
+  if (!VALID_FILE_TYPES.includes(fileType)) {
+    return null;
+  }
+
+  return {
+    filePath: pathToScan,
+    fileType: fileType as IacFileTypes,
+    fileContent: await loadFileContents(pathToScan, DEFAULT_ENCODING),
+  };
+}

src/cli/commands/test/iac-local-execution/file-parser.ts

@@ -0,0 +1,97 @@
+import * as hclToJson from 'hcl-to-json';
+import * as YAML from 'js-yaml';
+import {
+  EngineType,
+  IacFileParsed,
+  IacFileData,
+  ParsingResults,
+  IacFileParseFailure,
+} from './types';
+
+const REQUIRED_K8S_FIELDS = ['apiVersion', 'kind', 'metadata'];
+
+export async function parseFiles(
+  filesData: IacFileData[],
+): Promise<ParsingResults> {
+  const parsedFiles: Array<IacFileParsed> = [];
+  const failedFiles: Array<IacFileParseFailure> = [];
+  for (const fileData of filesData) {
+    try {
+      parsedFiles.push(...tryParseIacFile(fileData));
+    } catch (err) {
+      if (filesData.length === 1) throw err;
+      failedFiles.push(generateFailedParsedFile(fileData, err));
+    }
+  }
+
+  return {
+    parsedFiles,
+    failedFiles,
+  };
+}
+
+function generateFailedParsedFile(
+  { fileType, filePath, fileContent }: IacFileData,
+  err: Error,
+) {
+  return {
+    err,
+    failureReason: err.message,
+    fileType,
+    filePath,
+    fileContent,
+    engineType: null,
+    jsonContent: null,
+  };
+}
+
+function tryParseIacFile(fileData: IacFileData): Array<IacFileParsed> {
+  switch (fileData.fileType) {
+    case 'yaml':
+    case 'yml':
+    case 'json':
+      return tryParsingKubernetesFile(fileData);
+    case 'tf':
+      return tryParsingTerraformFile(fileData);
+    default:
+      throw new Error('Invalid IaC file');
+  }
+}
+
+function tryParsingKubernetesFile(fileData: IacFileData): IacFileParsed[] {
+  const yamlDocuments = YAML.safeLoadAll(fileData.fileContent);
+
+  return yamlDocuments.map((parsedYamlDocument, docId) => {
+    if (
+      REQUIRED_K8S_FIELDS.every((requiredField) =>
+        parsedYamlDocument.hasOwnProperty(requiredField),
+      )
+    ) {
+      return {
+        ...fileData,
+        jsonContent: parsedYamlDocument,
+        engineType: EngineType.Kubernetes,
+        docId,
+      };
+    } else {
+      throw new Error('Invalid K8s File!');
+    }
+  });
+}
+
+function tryParsingTerraformFile(fileData: IacFileData): Array<IacFileParsed> {
+  try {
+    // TODO: This parser does not fail on inavlid Terraform files! it is here temporarily.
+    // cloud-config team will replace it to a valid parser for the beta release.
+    const parsedData = hclToJson(fileData.fileContent);
+    return [
+      {
+        ...fileData,
+        jsonContent: parsedData,
+        engineType: EngineType.Terraform,
+      },
+    ];
+  } catch (err) {
+    throw new Error('Invalid Terraform File!');
+  }
+}

src/cli/commands/test/iac-local-execution/policy-engine.ts → src/cli/commands/test/iac-local-execution/file-scanner.ts

@@ -1,6 +1,6 @@
 import {
   OpaWasmInstance,
-  IacFileData,
+  IacFileParsed,
   IacFileScanResult,
   PolicyMetadata,
   EngineType,
@@ -9,14 +9,26 @@ import { loadPolicy } from '@open-policy-agent/opa-wasm';
 import * as fs from 'fs';
 import { getLocalCachePath, LOCAL_POLICY_ENGINE_DIR } from './local-cache';
 
-export async function getPolicyEngine(
-  engineType: EngineType,
-): Promise<PolicyEngine> {
+export async function scanFiles(
+  parsedFiles: Array<IacFileParsed>,
+): Promise<IacFileScanResult[]> {
+  // TODO: gracefully handle failed scans
+  const scanResults: IacFileScanResult[] = [];
+  for (const parsedFile of parsedFiles) {
+    const policyEngine = await getPolicyEngine(parsedFile.engineType);
+    const result = policyEngine.scanFile(parsedFile);
+    scanResults.push(result);
+  }
+
+  return scanResults;
+}
+
+async function getPolicyEngine(engineType: EngineType): Promise<PolicyEngine> {
   if (policyEngineCache[engineType]) {
     return policyEngineCache[engineType]!;
   }
-
-  policyEngineCache[engineType] = await buildPolicyEngine(engineType);
+  const policyEngine = await buildPolicyEngine(engineType);
+  policyEngineCache[engineType] = policyEngine;
   return policyEngineCache[engineType]!;
 }
 
@@ -62,7 +74,7 @@ class PolicyEngine {
     return this.opaWasmInstance.evaluate(data)[0].result;
   }
 
-  public scanFile(iacFile: IacFileData): IacFileScanResult {
+  public scanFile(iacFile: IacFileParsed): IacFileScanResult {
     try {
       const violatedPolicies = this.evaluate(iacFile.jsonContent);
       return {

src/cli/commands/test/iac-local-execution/index.ts

@@ -1,15 +1,9 @@
-import * as fs from 'fs';
-import { isLocalFolder } from '../../../../lib/detect';
-import { getFileType } from '../../../../lib/iac/iac-parser';
-import * as util from 'util';
-import { IacFileTypes } from '../../../../lib/iac/constants';
-import { IacFileScanResult, IacFileMetadata, IacFileData } from './types';
-import { getPolicyEngine } from './policy-engine';
-import { formatResults } from './results-formatter';
-import { tryParseIacFile } from './parsers';
 import { isLocalCacheExists, REQUIRED_LOCAL_CACHE_FILES } from './local-cache';
-
-const readFileContentsAsync = util.promisify(fs.readFile);
+import { loadFiles } from './file-loader';
+import { parseFiles } from './file-parser';
+import { scanFiles } from './file-scanner';
+import { formatScanResults } from './results-formatter';
+import { isLocalFolder } from '../../../../lib/detect';
 
 // this method executes the local processing engine and then formats the results to adapt with the CLI output.
 // the current version is dependent on files to be present locally which are not part of the source code.
@@ -22,61 +16,16 @@ export async function test(pathToScan: string, options) {
         '\n',
       )}`,
     );
-  // TODO: add support for proper typing of old TestResult interface.
-  const results = await localProcessing(pathToScan);
-  const formattedResults = formatResults(results, options);
-  const singleFileFormattedResult = formattedResults[0];
+  const filesToParse = await loadFiles(pathToScan);
+  const { parsedFiles, failedFiles } = await parseFiles(filesToParse);
+  const scannedFiles = await scanFiles(parsedFiles);
+  const formattedResults = formatScanResults(scannedFiles, options);
 
-  return singleFileFormattedResult as any;
-}
-
-async function localProcessing(
-  pathToScan: string,
-): Promise<IacFileScanResult[]> {
-  const filePathsToScan = await getFilePathsToScan(pathToScan);
-  const fileDataToScan = await parseFilesForScan(filePathsToScan);
-  const scanResults = await scanFilesForIssues(fileDataToScan);
-  return scanResults;
-}
-
-async function getFilePathsToScan(pathToScan): Promise<IacFileMetadata[]> {
   if (isLocalFolder(pathToScan)) {
-    throw new Error(
-      'IaC Experimental version does not support directory scan yet.',
-    );
+    // TODO: This mutation is here merely to support how the old/current directory scan printing works.
+    options.iacDirFiles = [...parsedFiles, ...failedFiles];
   }
 
-  return [
-    { filePath: pathToScan, fileType: getFileType(pathToScan) as IacFileTypes },
-  ];
-}
-
-async function parseFilesForScan(
-  filesMetadata: IacFileMetadata[],
-): Promise<IacFileData[]> {
-  const parsedFileData: Array<IacFileData> = [];
-  for (const fileMetadata of filesMetadata) {
-    const fileContent = await readFileContentsAsync(
-      fileMetadata.filePath,
-      'utf-8',
-    );
-    const parsedFiles = tryParseIacFile(fileMetadata, fileContent);
-    parsedFileData.push(...parsedFiles);
-  }
-
-  return parsedFileData;
-}
-
-async function scanFilesForIssues(
-  parsedFiles: Array<IacFileData>,
-): Promise<IacFileScanResult[]> {
-  // TODO: when adding dir support move implementation to queue.
-  // TODO: when adding dir support gracefully handle failed scans
-  return Promise.all(
-    parsedFiles.map(async (file) => {
-      const policyEngine = await getPolicyEngine(file.engineType);
-      const scanResults = policyEngine.scanFile(file);
-      return scanResults;
-    }),
-  );
+  // TODO: add support for proper typing of old TestResult interface.
+  return formattedResults as any;
 }