diff --git a/packages/build/src/log/messages/core_steps.js b/packages/build/src/log/messages/core_steps.js index bdc2501881..b83f2ed518 100644 --- a/packages/build/src/log/messages/core_steps.js +++ b/packages/build/src/log/messages/core_steps.js @@ -128,14 +128,19 @@ export const logSecretsScanSuccessMessage = function (logs, msg) { log(logs, msg, { color: THEME.highlightWords }) } -export const logSecretsScanFailBuildMessage = function ({ logs, scanResults, groupedResults }) { +export const logSecretsScanFailBuildMessage = function ({ + logs, + scanResults, + groupedResults, + enhancedScanShouldRunInActiveMode, +}) { const { secretMatches, enhancedSecretMatches } = groupedResults const secretMatchesKeys = Object.keys(secretMatches) const enhancedSecretMatchesKeys = Object.keys(enhancedSecretMatches) logErrorSubHeader( logs, - `Scanning complete. ${scanResults.scannedFilesCount} file(s) scanned. Secrets scanning found ${secretMatchesKeys.length} instance(s) of secrets${enhancedSecretMatchesKeys.length > 0 ? ` and ${enhancedSecretMatchesKeys.length} instance(s) of likely secrets` : ''} in build output or repo code.\n`, + `Scanning complete. ${scanResults.scannedFilesCount} file(s) scanned. Secrets scanning found ${secretMatchesKeys.length} instance(s) of secrets${enhancedSecretMatchesKeys.length > 0 && enhancedScanShouldRunInActiveMode ? ` and ${enhancedSecretMatchesKeys.length} instance(s) of likely secrets` : ''} in build output or repo code.\n`, ) // Explicit secret matches @@ -162,28 +167,30 @@ export const logSecretsScanFailBuildMessage = function ({ logs, scanResults, gro ) } - // Likely secret matches from enhanced scan - enhancedSecretMatchesKeys.forEach((key, index) => { - logError(logs, `${index === 0 && secretMatchesKeys.length ? '\n' : ''}"${key}***" detected as a likely secret:`) - - enhancedSecretMatches[key] - .sort((a, b) => { - return a.file > b.file ? 0 : 1 - }) - .forEach(({ lineNumber, file }) => { - logError(logs, `found value at line ${lineNumber} in ${file}`, { indent: true }) - }) - }) + if (enhancedScanShouldRunInActiveMode) { + // Likely secret matches from enhanced scan + enhancedSecretMatchesKeys.forEach((key, index) => { + logError(logs, `${index === 0 && secretMatchesKeys.length ? '\n' : ''}"${key}***" detected as a likely secret:`) + + enhancedSecretMatches[key] + .sort((a, b) => { + return a.file > b.file ? 0 : 1 + }) + .forEach(({ lineNumber, file }) => { + logError(logs, `found value at line ${lineNumber} in ${file}`, { indent: true }) + }) + }) - if (enhancedSecretMatchesKeys.length) { - logError( - logs, - `\nTo prevent exposing secrets, the build will fail until these likely secret values are not found in build output or repo files.`, - ) - logError( - logs, - `\nIf these are expected, use ENHANCED_SECRETS_SCAN_OMIT_VALUES, or ENHANCED_SECRETS_SCAN_ENABLED to prevent detecting.`, - ) + if (enhancedSecretMatchesKeys.length) { + logError( + logs, + `\nTo prevent exposing secrets, the build will fail until these likely secret values are not found in build output or repo files.`, + ) + logError( + logs, + `\nIf these are expected, use ENHANCED_SECRETS_SCAN_OMIT_VALUES, or ENHANCED_SECRETS_SCAN_ENABLED to prevent detecting.`, + ) + } } logError( diff --git a/packages/build/src/plugins_core/secrets_scanning/index.ts b/packages/build/src/plugins_core/secrets_scanning/index.ts index 455497fa8e..4c2e8ceaf8 100644 --- a/packages/build/src/plugins_core/secrets_scanning/index.ts +++ b/packages/build/src/plugins_core/secrets_scanning/index.ts @@ -39,6 +39,10 @@ const coreStep: CoreStepFunction = async function ({ const passedSecretKeys = (explicitSecretKeys || '').split(',') const envVars = netlifyConfig.build.environment as Record + // When the flag is disabled, we may still run the scan if a secrets scan would otherwise take place anyway + // In this case, we hide any output to the user and simply gather the information in our logs + const enhancedScanShouldRunInActiveMode = featureFlags?.enhanced_secret_scan_impacts_builds ?? false + const useMinimalChunks = featureFlags?.secret_scanning_minimal_chunks systemLog?.({ passedSecretKeys, buildDir }) @@ -56,15 +60,17 @@ const coreStep: CoreStepFunction = async function ({ log(logs, `SECRETS_SCAN_OMIT_PATHS override option set to: ${envVars['SECRETS_SCAN_OMIT_PATHS']}\n`) } const enhancedScanningEnabledInEnv = isEnhancedSecretsScanningEnabled(envVars) - if (enhancedSecretScan && !enhancedScanningEnabledInEnv) { + const enhancedScanConfigured = enhancedSecretScan && enhancedScanningEnabledInEnv + if (enhancedSecretScan && enhancedScanShouldRunInActiveMode && !enhancedScanningEnabledInEnv) { logSecretsScanSkipMessage( logs, 'Enhanced secrets detection disabled via ENHANCED_SECRETS_SCAN_ENABLED flag set to false.', ) } + if ( - enhancedSecretScan && - enhancedScanningEnabledInEnv && + enhancedScanShouldRunInActiveMode && + enhancedScanConfigured && envVars['ENHANCED_SECRETS_SCAN_OMIT_VALUES'] !== undefined ) { log( @@ -75,7 +81,11 @@ const coreStep: CoreStepFunction = async function ({ const keysToSearchFor = getSecretKeysToScanFor(envVars, passedSecretKeys) - if (keysToSearchFor.length === 0 && !enhancedSecretScan) { + // In passive mode, only run the enhanced scan if we have explicit secret keys + const enhancedScanShouldRun = enhancedScanShouldRunInActiveMode + ? enhancedScanConfigured + : enhancedScanConfigured && keysToSearchFor.length > 0 + if (keysToSearchFor.length === 0 && !enhancedScanShouldRun) { logSecretsScanSkipMessage( logs, 'Secrets scanning skipped because no env vars marked as secret are set to non-empty/non-trivial values or they are all omitted with SECRETS_SCAN_OMIT_KEYS env var setting.', @@ -109,7 +119,7 @@ const coreStep: CoreStepFunction = async function ({ keys: keysToSearchFor, base: buildDir as string, filePaths, - enhancedScanning: enhancedSecretScan && enhancedScanningEnabledInEnv, + enhancedScanning: enhancedScanShouldRun, omitValuesFromEnhancedScan: getOmitValuesFromEnhancedScanForEnhancedScanFromEnv(envVars), useMinimalChunks, }) @@ -125,7 +135,8 @@ const coreStep: CoreStepFunction = async function ({ secretsFilesCount: scanResults.scannedFilesCount, keysToSearchFor, enhancedPrefixMatches: enhancedSecretMatches.length ? enhancedSecretMatches.map((match) => match.key) : [], - enhancedScanning: enhancedSecretScan && enhancedScanningEnabledInEnv, + enhancedScanning: enhancedScanShouldRun, + enhancedScanActiveMode: enhancedScanShouldRunInActiveMode, } systemLog?.(attributesForLogsAndSpan) @@ -138,12 +149,17 @@ const coreStep: CoreStepFunction = async function ({ const secretScanResult: SecretScanResult = { scannedFilesCount: scanResults?.scannedFilesCount ?? 0, secretsScanMatches: secretMatches ?? [], - enhancedSecretsScanMatches: enhancedSecretMatches ?? [], + enhancedSecretsScanMatches: + enhancedScanShouldRunInActiveMode && enhancedSecretMatches ? enhancedSecretMatches : [], } reportValidations({ api, secretScanResult, deployId, systemLog }) } - if (!scanResults || scanResults.matches.length === 0) { + if ( + !scanResults || + scanResults.matches.length === 0 || + (!enhancedScanShouldRunInActiveMode && !secretMatches?.length) + ) { logSecretsScanSuccessMessage( logs, `Secrets scanning complete. ${scanResults?.scannedFilesCount} file(s) scanned. No secrets detected in build output or repo code!`, @@ -157,6 +173,7 @@ const coreStep: CoreStepFunction = async function ({ logs, scanResults, groupedResults: groupScanResultsByKeyAndScanType(scanResults), + enhancedScanShouldRunInActiveMode, }) const error = new Error(`Secrets scanning found secrets in build.`) diff --git a/packages/build/tests/secrets_scanning/tests.js b/packages/build/tests/secrets_scanning/tests.js index b3b469ec65..7f6197bfa6 100644 --- a/packages/build/tests/secrets_scanning/tests.js +++ b/packages/build/tests/secrets_scanning/tests.js @@ -258,11 +258,17 @@ for (const { testPrefix, featureFlags } of [ t.true(output.includes(`No secrets detected in build output or repo code!`)) }) - // Enhanced secret scanning + // Enhanced secret scanning with enhanced_secret_scan_impacts_builds enabled test(testPrefix + 'secrets scanning, enhanced scan should not run when disabled', async (t) => { const { requests } = await new Fixture('./fixtures/src_scanning_disabled') - .withFlags({ debug: false, enhancedSecretScan: true, deployId: 'test', token: 'test', featureFlags }) + .withFlags({ + debug: false, + enhancedSecretScan: true, + deployId: 'test', + token: 'test', + featureFlags: { ...featureFlags, enhanced_secret_scan_impacts_builds: true }, + }) .runBuildServer({ path: '/api/v1/deploys/test/validations_report' }) t.true(requests.length === 0) }) @@ -278,7 +284,7 @@ for (const { testPrefix, featureFlags } of [ enhancedSecretScan: true, deployId: 'test', token: 'test', - featureFlags, + featureFlags: { ...featureFlags, enhanced_secret_scan_impacts_builds: true }, }) .runBuildServer({ path: '/api/v1/deploys/test/validations_report' }) @@ -293,11 +299,11 @@ for (const { testPrefix, featureFlags } of [ const { requests } = await new Fixture('./fixtures/src_scanning_likely_enhanced_scan_secrets_disabled') .withFlags({ debug: false, - explicitSecretKeys: '', + explicitSecretKeys: 'ENV_VAR_1', enhancedSecretScan: true, deployId: 'test', token: 'test', - featureFlags, + featureFlags: { ...featureFlags, enhanced_secret_scan_impacts_builds: true }, }) .runBuildServer({ path: '/api/v1/deploys/test/validations_report' }) t.true(requests.length === 1) @@ -316,7 +322,7 @@ for (const { testPrefix, featureFlags } of [ enhancedSecretScan: true, deployId: 'test', token: 'test', - featureFlags, + featureFlags: { ...featureFlags, enhanced_secret_scan_impacts_builds: true }, }) .runBuildServer({ path: '/api/v1/deploys/test/validations_report' }) @@ -337,7 +343,7 @@ for (const { testPrefix, featureFlags } of [ enhancedSecretScan: false, deployId: 'test', token: 'test', - featureFlags, + featureFlags: { ...featureFlags, enhanced_secret_scan_impacts_builds: true }, }) .runBuildServer({ path: '/api/v1/deploys/test/validations_report' }) @@ -353,7 +359,7 @@ for (const { testPrefix, featureFlags } of [ enhancedSecretScan: true, deployId: 'test', token: 'test', - featureFlags, + featureFlags: { ...featureFlags, enhanced_secret_scan_impacts_builds: true }, }) .runBuildServer({ path: '/api/v1/deploys/test/validations_report' }) @@ -367,7 +373,13 @@ for (const { testPrefix, featureFlags } of [ test(testPrefix + 'secrets scanning, should not find secrets in files without known prefixes', async (t) => { const { requests } = await new Fixture('./fixtures/src_scanning_no_likely_enhanced_scan_secrets', featureFlags) - .withFlags({ debug: false, enhancedSecretScan: true, deployId: 'test', token: 'test' }) + .withFlags({ + debug: false, + enhancedSecretScan: true, + deployId: 'test', + token: 'test', + featureFlags: { ...featureFlags, enhanced_secret_scan_impacts_builds: true }, + }) .runBuildServer({ path: '/api/v1/deploys/test/validations_report' }) t.true(requests.length === 1) @@ -389,7 +401,7 @@ for (const { testPrefix, featureFlags } of [ enhancedSecretScan: true, deployId: 'test', token: 'test', - featureFlags, + featureFlags: { ...featureFlags, enhanced_secret_scan_impacts_builds: true }, }) .runBuildServer({ path: '/api/v1/deploys/test/validations_report' }) @@ -414,7 +426,7 @@ for (const { testPrefix, featureFlags } of [ enhancedSecretScan: true, deployId: 'test', token: 'test', - featureFlags, + featureFlags: { ...featureFlags, enhanced_secret_scan_impacts_builds: true }, }) .runBuildServer({ path: '/api/v1/deploys/test/validations_report' }) @@ -443,7 +455,7 @@ for (const { testPrefix, featureFlags } of [ enhancedSecretScan: true, deployId: 'test', token: 'test', - featureFlags, + featureFlags: { ...featureFlags, enhanced_secret_scan_impacts_builds: true }, }) .runBuildServer({ path: '/api/v1/deploys/test/validations_report' }) @@ -463,12 +475,108 @@ for (const { testPrefix, featureFlags } of [ debug: false, explicitSecretKeys: '', enhancedSecretScan: true, - featureFlags, + featureFlags: { ...featureFlags, enhanced_secret_scan_impacts_builds: true }, }) .runBuildProgrammatic() // Severity code of 2 is user error t.is(severityCode, 2) }) + + // enhanced scanning enabled, but without impact to builds + + test( + testPrefix + + 'secrets scanning, should not log enhanced scan info when enhanced_secret_scan_impacts_builds is false', + async (t) => { + const { output } = await new Fixture('./fixtures/src_scanning_likely_enhanced_scan_secrets') + .withFlags({ + debug: false, + enhancedSecretScan: true, + featureFlags: { ...featureFlags, enhanced_secret_scan_impacts_builds: false }, + deployId: 'test', + token: 'test', + }) + .runBuildServer({ path: '/api/v1/deploys/test/validations_report' }) + + const normalizedOutput = normalizeOutput(output) + t.false(normalizedOutput.includes('detected as a likely secret')) + }, + ) + + test( + testPrefix + + 'secrets scanning, should not fail build when enhanced scan finds likely secrets but enhanced_secret_scan_impacts_builds is false', + async (t) => { + const { severityCode } = await new Fixture('./fixtures/src_scanning_likely_enhanced_scan_secrets') + .withFlags({ + debug: false, + enhancedSecretScan: true, + featureFlags: { ...featureFlags, enhanced_secret_scan_impacts_builds: false }, + }) + .runBuildProgrammatic() + + // Severity code of 0 means success, 2 would be user error + t.is(severityCode, 0) + }, + ) + + test( + testPrefix + + 'secrets scanning, should not log omit values message when enhanced_secret_scan_impacts_builds is false', + async (t) => { + const { output } = await new Fixture('./fixtures/src_scanning_likely_enhanced_scan_secrets_omitted') + .withFlags({ + debug: false, + enhancedSecretScan: true, + featureFlags: { ...featureFlags, enhanced_secret_scan_impacts_builds: false }, + deployId: 'test', + token: 'test', + }) + .runBuildServer({ path: '/api/v1/deploys/test/validations_report' }) + + t.false(normalizeOutput(output).includes('ENHANCED_SECRETS_SCAN_OMIT_VALUES')) + }, + ) + + test( + testPrefix + 'secrets scanning, should run enhanced scan in passive mode when explicit keys are present', + async (t) => { + const { requests } = await new Fixture('./fixtures/src_scanning_env_vars_set_non_empty') + .withFlags({ + debug: false, + explicitSecretKeys: 'ENV_VAR_1', + enhancedSecretScan: true, + featureFlags: { ...featureFlags, enhanced_secret_scan_impacts_builds: false }, + deployId: 'test', + token: 'test', + }) + .runBuildServer({ path: '/api/v1/deploys/test/validations_report' }) + + t.true(requests.length === 1) + const request = requests[0] + t.is(request.url, '/api/v1/deploys/test/validations_report') + t.truthy(request.body.secrets_scan.scannedFilesCount) + t.truthy(request.body.secrets_scan.enhancedSecretsScanMatches) + }, + ) + + test( + testPrefix + 'secrets scanning, should not run enhanced scan in passive mode when no explicit keys', + async (t) => { + const { requests } = await new Fixture('./fixtures/src_scanning_likely_enhanced_scan_secrets') + .withFlags({ + debug: false, + explicitSecretKeys: '', + enhancedSecretScan: true, + featureFlags: { ...featureFlags, enhanced_secret_scan_impacts_builds: false }, + deployId: 'test', + token: 'test', + }) + .runBuildServer({ path: '/api/v1/deploys/test/validations_report' }) + + t.true(requests.length === 0) + }, + ) ;(featureFlags.secret_scanning_minimal_chunks ? test : test.skip)( testPrefix + 'does not crash if line in scanned file exceed available memory', async (t) => {