feat: allow enhanced scan to run without impacting builds

packages/build/src/log/messages/core_steps.js

@@ -128,14 +128,19 @@ export const logSecretsScanSuccessMessage = function (logs, msg) {
   log(logs, msg, { color: THEME.highlightWords })
 }
 
-export const logSecretsScanFailBuildMessage = function ({ logs, scanResults, groupedResults }) {
+export const logSecretsScanFailBuildMessage = function ({
+  logs,
+  scanResults,
+  groupedResults,
+  enhancedScanShouldRunInActiveMode,
+}) {
   const { secretMatches, enhancedSecretMatches } = groupedResults
   const secretMatchesKeys = Object.keys(secretMatches)
   const enhancedSecretMatchesKeys = Object.keys(enhancedSecretMatches)
 
   logErrorSubHeader(
     logs,
-    `Scanning complete. ${scanResults.scannedFilesCount} file(s) scanned. Secrets scanning found ${secretMatchesKeys.length} instance(s) of secrets${enhancedSecretMatchesKeys.length > 0 ? ` and ${enhancedSecretMatchesKeys.length} instance(s) of likely secrets` : ''} in build output or repo code.\n`,
+    `Scanning complete. ${scanResults.scannedFilesCount} file(s) scanned. Secrets scanning found ${secretMatchesKeys.length} instance(s) of secrets${enhancedSecretMatchesKeys.length > 0 && enhancedScanShouldRunInActiveMode ? ` and ${enhancedSecretMatchesKeys.length} instance(s) of likely secrets` : ''} in build output or repo code.\n`,
   )
 
   // Explicit secret matches
@@ -162,28 +167,30 @@ export const logSecretsScanFailBuildMessage = function ({ logs, scanResults, gro
     )
   }
 
-  // Likely secret matches from enhanced scan
-  enhancedSecretMatchesKeys.forEach((key, index) => {
-    logError(logs, `${index === 0 && secretMatchesKeys.length ? '\n' : ''}"${key}***" detected as a likely secret:`)
-
-    enhancedSecretMatches[key]
-      .sort((a, b) => {
-        return a.file > b.file ? 0 : 1
-      })
-      .forEach(({ lineNumber, file }) => {
-        logError(logs, `found value at line ${lineNumber} in ${file}`, { indent: true })
-      })
-  })
+  if (enhancedScanShouldRunInActiveMode) {
+    // Likely secret matches from enhanced scan
+    enhancedSecretMatchesKeys.forEach((key, index) => {
+      logError(logs, `${index === 0 && secretMatchesKeys.length ? '\n' : ''}"${key}***" detected as a likely secret:`)
+
+      enhancedSecretMatches[key]
+        .sort((a, b) => {
+          return a.file > b.file ? 0 : 1
+        })
+        .forEach(({ lineNumber, file }) => {
+          logError(logs, `found value at line ${lineNumber} in ${file}`, { indent: true })
+        })
+    })
 
-  if (enhancedSecretMatchesKeys.length) {
-    logError(
-      logs,
-      `\nTo prevent exposing secrets, the build will fail until these likely secret values are not found in build output or repo files.`,
-    )
-    logError(
-      logs,
-      `\nIf these are expected, use ENHANCED_SECRETS_SCAN_OMIT_VALUES, or ENHANCED_SECRETS_SCAN_ENABLED to prevent detecting.`,
-    )
+    if (enhancedSecretMatchesKeys.length) {
+      logError(
+        logs,
+        `\nTo prevent exposing secrets, the build will fail until these likely secret values are not found in build output or repo files.`,
+      )
+      logError(
+        logs,
+        `\nIf these are expected, use ENHANCED_SECRETS_SCAN_OMIT_VALUES, or ENHANCED_SECRETS_SCAN_ENABLED to prevent detecting.`,
+      )
+    }
   }
 
   logError(

packages/build/src/plugins_core/secrets_scanning/index.ts

@@ -39,6 +39,10 @@ const coreStep: CoreStepFunction = async function ({
 
   const passedSecretKeys = (explicitSecretKeys || '').split(',')
   const envVars = netlifyConfig.build.environment as Record<string, unknown>
+  // When the flag is disabled, we may still run the scan if a secrets scan would otherwise take place anyway
+  // In this case, we hide any output to the user and simply gather the information in our logs
+  const enhancedScanShouldRunInActiveMode = featureFlags?.enhanced_secret_scan_impacts_builds ?? false
+
   const useMinimalChunks = featureFlags?.secret_scanning_minimal_chunks
 
   systemLog?.({ passedSecretKeys, buildDir })
@@ -56,15 +60,17 @@ const coreStep: CoreStepFunction = async function ({
     log(logs, `SECRETS_SCAN_OMIT_PATHS override option set to: ${envVars['SECRETS_SCAN_OMIT_PATHS']}\n`)
   }
   const enhancedScanningEnabledInEnv = isEnhancedSecretsScanningEnabled(envVars)
-  if (enhancedSecretScan && !enhancedScanningEnabledInEnv) {
+  const enhancedScanConfigured = enhancedSecretScan && enhancedScanningEnabledInEnv
+  if (enhancedSecretScan && enhancedScanShouldRunInActiveMode && !enhancedScanningEnabledInEnv) {
     logSecretsScanSkipMessage(
       logs,
       'Enhanced secrets detection disabled via ENHANCED_SECRETS_SCAN_ENABLED flag set to false.',
     )
   }
+
   if (
-    enhancedSecretScan &&
-    enhancedScanningEnabledInEnv &&
+    enhancedScanShouldRunInActiveMode &&
+    enhancedScanConfigured &&
     envVars['ENHANCED_SECRETS_SCAN_OMIT_VALUES'] !== undefined
   ) {
     log(
@@ -75,7 +81,11 @@ const coreStep: CoreStepFunction = async function ({
 
   const keysToSearchFor = getSecretKeysToScanFor(envVars, passedSecretKeys)
 
-  if (keysToSearchFor.length === 0 && !enhancedSecretScan) {
+  // In passive mode, only run the enhanced scan if we have explicit secret keys
+  const enhancedScanShouldRun = enhancedScanShouldRunInActiveMode
+    ? enhancedScanConfigured
+    : enhancedScanConfigured && keysToSearchFor.length > 0
+  if (keysToSearchFor.length === 0 && !enhancedScanShouldRun) {
     logSecretsScanSkipMessage(
       logs,
       'Secrets scanning skipped because no env vars marked as secret are set to non-empty/non-trivial values or they are all omitted with SECRETS_SCAN_OMIT_KEYS env var setting.',
@@ -109,7 +119,7 @@ const coreStep: CoreStepFunction = async function ({
         keys: keysToSearchFor,
         base: buildDir as string,
         filePaths,
-        enhancedScanning: enhancedSecretScan && enhancedScanningEnabledInEnv,
+        enhancedScanning: enhancedScanShouldRun,
         omitValuesFromEnhancedScan: getOmitValuesFromEnhancedScanForEnhancedScanFromEnv(envVars),
         useMinimalChunks,
       })
@@ -125,7 +135,8 @@ const coreStep: CoreStepFunction = async function ({
         secretsFilesCount: scanResults.scannedFilesCount,
         keysToSearchFor,
         enhancedPrefixMatches: enhancedSecretMatches.length ? enhancedSecretMatches.map((match) => match.key) : [],
-        enhancedScanning: enhancedSecretScan && enhancedScanningEnabledInEnv,
+        enhancedScanning: enhancedScanShouldRun,
+        enhancedScanActiveMode: enhancedScanShouldRunInActiveMode,
       }
 
       systemLog?.(attributesForLogsAndSpan)
@@ -138,12 +149,17 @@ const coreStep: CoreStepFunction = async function ({
     const secretScanResult: SecretScanResult = {
       scannedFilesCount: scanResults?.scannedFilesCount ?? 0,
       secretsScanMatches: secretMatches ?? [],
-      enhancedSecretsScanMatches: enhancedSecretMatches ?? [],
+      enhancedSecretsScanMatches:
+        enhancedScanShouldRunInActiveMode && enhancedSecretMatches ? enhancedSecretMatches : [],
     }
     reportValidations({ api, secretScanResult, deployId, systemLog })
   }
 
-  if (!scanResults || scanResults.matches.length === 0) {
+  if (
+    !scanResults ||
+    scanResults.matches.length === 0 ||
+    (!enhancedScanShouldRunInActiveMode && !secretMatches?.length)
+  ) {
     logSecretsScanSuccessMessage(
       logs,
       `Secrets scanning complete. ${scanResults?.scannedFilesCount} file(s) scanned. No secrets detected in build output or repo code!`,
@@ -157,6 +173,7 @@ const coreStep: CoreStepFunction = async function ({
     logs,
     scanResults,
     groupedResults: groupScanResultsByKeyAndScanType(scanResults),
+    enhancedScanShouldRunInActiveMode,
   })
 
   const error = new Error(`Secrets scanning found secrets in build.`)

packages/build/tests/secrets_scanning/tests.js

@@ -258,11 +258,17 @@ for (const { testPrefix, featureFlags } of [
     t.true(output.includes(`No secrets detected in build output or repo code!`))
   })
 
-  // Enhanced secret scanning
+  // Enhanced secret scanning with enhanced_secret_scan_impacts_builds enabled
 
   test(testPrefix + 'secrets scanning, enhanced scan should not run when disabled', async (t) => {
     const { requests } = await new Fixture('./fixtures/src_scanning_disabled')
-      .withFlags({ debug: false, enhancedSecretScan: true, deployId: 'test', token: 'test', featureFlags })
+      .withFlags({
+        debug: false,
+        enhancedSecretScan: true,
+        deployId: 'test',
+        token: 'test',
+        featureFlags: { ...featureFlags, enhanced_secret_scan_impacts_builds: true },
+      })
       .runBuildServer({ path: '/api/v1/deploys/test/validations_report' })
     t.true(requests.length === 0)
   })
@@ -278,7 +284,7 @@ for (const { testPrefix, featureFlags } of [
           enhancedSecretScan: true,
           deployId: 'test',
           token: 'test',
-          featureFlags,
+          featureFlags: { ...featureFlags, enhanced_secret_scan_impacts_builds: true },
         })
         .runBuildServer({ path: '/api/v1/deploys/test/validations_report' })
 
@@ -293,11 +299,11 @@ for (const { testPrefix, featureFlags } of [
       const { requests } = await new Fixture('./fixtures/src_scanning_likely_enhanced_scan_secrets_disabled')
         .withFlags({
           debug: false,
-          explicitSecretKeys: '',
+          explicitSecretKeys: 'ENV_VAR_1',
           enhancedSecretScan: true,
           deployId: 'test',
           token: 'test',
-          featureFlags,
+          featureFlags: { ...featureFlags, enhanced_secret_scan_impacts_builds: true },
         })
         .runBuildServer({ path: '/api/v1/deploys/test/validations_report' })
       t.true(requests.length === 1)
@@ -316,7 +322,7 @@ for (const { testPrefix, featureFlags } of [
           enhancedSecretScan: true,
           deployId: 'test',
           token: 'test',
-          featureFlags,
+          featureFlags: { ...featureFlags, enhanced_secret_scan_impacts_builds: true },
         })
         .runBuildServer({ path: '/api/v1/deploys/test/validations_report' })
 
@@ -337,7 +343,7 @@ for (const { testPrefix, featureFlags } of [
           enhancedSecretScan: false,
           deployId: 'test',
           token: 'test',
-          featureFlags,
+          featureFlags: { ...featureFlags, enhanced_secret_scan_impacts_builds: true },
         })
         .runBuildServer({ path: '/api/v1/deploys/test/validations_report' })
 
@@ -353,7 +359,7 @@ for (const { testPrefix, featureFlags } of [
         enhancedSecretScan: true,
         deployId: 'test',
         token: 'test',
-        featureFlags,
+        featureFlags: { ...featureFlags, enhanced_secret_scan_impacts_builds: true },
       })
       .runBuildServer({ path: '/api/v1/deploys/test/validations_report' })
 
@@ -367,7 +373,13 @@ for (const { testPrefix, featureFlags } of [
 
   test(testPrefix + 'secrets scanning, should not find secrets in files without known prefixes', async (t) => {
     const { requests } = await new Fixture('./fixtures/src_scanning_no_likely_enhanced_scan_secrets', featureFlags)
-      .withFlags({ debug: false, enhancedSecretScan: true, deployId: 'test', token: 'test' })
+      .withFlags({
+        debug: false,
+        enhancedSecretScan: true,
+        deployId: 'test',
+        token: 'test',
+        featureFlags: { ...featureFlags, enhanced_secret_scan_impacts_builds: true },
+      })
       .runBuildServer({ path: '/api/v1/deploys/test/validations_report' })
 
     t.true(requests.length === 1)
@@ -389,7 +401,7 @@ for (const { testPrefix, featureFlags } of [
           enhancedSecretScan: true,
           deployId: 'test',
           token: 'test',
-          featureFlags,
+          featureFlags: { ...featureFlags, enhanced_secret_scan_impacts_builds: true },
         })
         .runBuildServer({ path: '/api/v1/deploys/test/validations_report' })
 
@@ -414,7 +426,7 @@ for (const { testPrefix, featureFlags } of [
           enhancedSecretScan: true,
           deployId: 'test',
           token: 'test',
-          featureFlags,
+          featureFlags: { ...featureFlags, enhanced_secret_scan_impacts_builds: true },
         })
         .runBuildServer({ path: '/api/v1/deploys/test/validations_report' })
 
@@ -443,7 +455,7 @@ for (const { testPrefix, featureFlags } of [
           enhancedSecretScan: true,
           deployId: 'test',
           token: 'test',
-          featureFlags,
+          featureFlags: { ...featureFlags, enhanced_secret_scan_impacts_builds: true },
         })
         .runBuildServer({ path: '/api/v1/deploys/test/validations_report' })
 
@@ -463,12 +475,108 @@ for (const { testPrefix, featureFlags } of [
         debug: false,
         explicitSecretKeys: '',
         enhancedSecretScan: true,
-        featureFlags,
+        featureFlags: { ...featureFlags, enhanced_secret_scan_impacts_builds: true },
       })
       .runBuildProgrammatic()
     // Severity code of 2 is user error
     t.is(severityCode, 2)
   })
+
+  // enhanced scanning enabled, but without impact to builds
+
+  test(
+    testPrefix +
+      'secrets scanning, should not log enhanced scan info when enhanced_secret_scan_impacts_builds is false',
+    async (t) => {
+      const { output } = await new Fixture('./fixtures/src_scanning_likely_enhanced_scan_secrets')
+        .withFlags({
+          debug: false,
+          enhancedSecretScan: true,
+          featureFlags: { ...featureFlags, enhanced_secret_scan_impacts_builds: false },
+          deployId: 'test',
+          token: 'test',
+        })
+        .runBuildServer({ path: '/api/v1/deploys/test/validations_report' })
+
+      const normalizedOutput = normalizeOutput(output)
+      t.false(normalizedOutput.includes('detected as a likely secret'))
+    },
+  )
+
+  test(
+    testPrefix +
+      'secrets scanning, should not fail build when enhanced scan finds likely secrets but enhanced_secret_scan_impacts_builds is false',
+    async (t) => {
+      const { severityCode } = await new Fixture('./fixtures/src_scanning_likely_enhanced_scan_secrets')
+        .withFlags({
+          debug: false,
+          enhancedSecretScan: true,
+          featureFlags: { ...featureFlags, enhanced_secret_scan_impacts_builds: false },
+        })
+        .runBuildProgrammatic()
+
+      // Severity code of 0 means success, 2 would be user error
+      t.is(severityCode, 0)
+    },
+  )
+
+  test(
+    testPrefix +
+      'secrets scanning, should not log omit values message when enhanced_secret_scan_impacts_builds is false',
+    async (t) => {
+      const { output } = await new Fixture('./fixtures/src_scanning_likely_enhanced_scan_secrets_omitted')
+        .withFlags({
+          debug: false,
+          enhancedSecretScan: true,
+          featureFlags: { ...featureFlags, enhanced_secret_scan_impacts_builds: false },
+          deployId: 'test',
+          token: 'test',
+        })
+        .runBuildServer({ path: '/api/v1/deploys/test/validations_report' })
+
+      t.false(normalizeOutput(output).includes('ENHANCED_SECRETS_SCAN_OMIT_VALUES'))
+    },
+  )
+
+  test(
+    testPrefix + 'secrets scanning, should run enhanced scan in passive mode when explicit keys are present',
+    async (t) => {
+      const { requests } = await new Fixture('./fixtures/src_scanning_env_vars_set_non_empty')
+        .withFlags({
+          debug: false,
+          explicitSecretKeys: 'ENV_VAR_1',
+          enhancedSecretScan: true,
+          featureFlags: { ...featureFlags, enhanced_secret_scan_impacts_builds: false },
+          deployId: 'test',
+          token: 'test',
+        })
+        .runBuildServer({ path: '/api/v1/deploys/test/validations_report' })
+
+      t.true(requests.length === 1)
+      const request = requests[0]
+      t.is(request.url, '/api/v1/deploys/test/validations_report')
+      t.truthy(request.body.secrets_scan.scannedFilesCount)
+      t.truthy(request.body.secrets_scan.enhancedSecretsScanMatches)
+    },
+  )
+
+  test(
+    testPrefix + 'secrets scanning, should not run enhanced scan in passive mode when no explicit keys',
+    async (t) => {
+      const { requests } = await new Fixture('./fixtures/src_scanning_likely_enhanced_scan_secrets')
+        .withFlags({
+          debug: false,
+          explicitSecretKeys: '',
+          enhancedSecretScan: true,
+          featureFlags: { ...featureFlags, enhanced_secret_scan_impacts_builds: false },
+          deployId: 'test',
+          token: 'test',
+        })
+        .runBuildServer({ path: '/api/v1/deploys/test/validations_report' })
+
+      t.true(requests.length === 0)
+    },
+  )
   ;(featureFlags.secret_scanning_minimal_chunks ? test : test.skip)(
     testPrefix + 'does not crash if line in scanned file exceed available memory',
     async (t) => {