feat(aap): Extended request data collection via rules (#6275)

Author: uurien

index.d.ts

@@ -786,6 +786,8 @@ declare namespace tracer {
 
         /** Whether to enable request body collection on RASP event
          * @default false
+         *
+         * @deprecated Use UI and Remote Configuration to enable extended data collection
          */
         bodyCollection?: boolean
       },
@@ -810,20 +812,28 @@ declare namespace tracer {
       },
       /**
        * Configuration for extended headers collection tied to security events
+       *
+       * @deprecated Use UI and Remote Configuration to enable extended data collection
        */
       extendedHeadersCollection?: {
         /** Whether to enable extended headers collection
          * @default false
+         *
+         * @deprecated Use UI and Remote Configuration to enable extended data collection
          */
         enabled: boolean,
 
         /** Whether to redact collected headers
          * @default true
+         *
+         * @deprecated Use UI and Remote Configuration to enable extended data collection
          */
         redaction: boolean,
 
         /** Specifies the maximum number of headers collected.
          * @default 50
+         *
+         * @deprecated Use UI and Remote Configuration to enable extended data collection
          */
         maxHeaders: number,
       }

packages/dd-trace/src/appsec/reporter.js

@@ -38,14 +38,20 @@ const config = {
 
 const metricsQueue = new Map()
 
+const extendedDataCollectionRequest = new WeakMap()
+
 // following header lists are ordered in the same way the spec orders them, it doesn't matter but it's easier to compare
 const contentHeaderList = [
   'content-length',
-  'content-type',
   'content-encoding',
   'content-language'
 ]
 
+const responseHeaderList = [
+  ...contentHeaderList,
+  'content-type'
+]
+
 const identificationHeaders = [
   'x-amzn-trace-id',
   'cloudfront-viewer-ja3-fingerprint',
@@ -75,15 +81,27 @@ const requestHeadersList = [
   ...identificationHeaders
 ]
 
+const redactedHeadersList = [
+  'authorization',
+  'proxy-authorization',
+  'www-authenticate',
+  'proxy-authenticate',
+  'authentication-info',
+  'proxy-authentication-info',
+  'cookie',
+  'set-cookie'
+]
+
 // these request headers are always collected - it breaks the expected spec orders
 const REQUEST_HEADERS_MAP = mapHeaderAndTags(requestHeadersList, REQUEST_HEADER_TAG_PREFIX)
 
 const EVENT_HEADERS_MAP = mapHeaderAndTags(eventHeadersList, REQUEST_HEADER_TAG_PREFIX)
 
-const RESPONSE_HEADERS_MAP = mapHeaderAndTags(contentHeaderList, RESPONSE_HEADER_TAG_PREFIX)
+const RESPONSE_HEADERS_MAP = mapHeaderAndTags(responseHeaderList, RESPONSE_HEADER_TAG_PREFIX)
 
 const NON_EXTENDED_REQUEST_HEADERS = new Set([...requestHeadersList, ...eventHeadersList])
-const NON_EXTENDED_RESPONSE_HEADERS = new Set(contentHeaderList)
+const NON_EXTENDED_RESPONSE_HEADERS = new Set(responseHeaderList)
+const REDACTED_HEADERS = new Set(redactedHeadersList)
 
 function init (_config) {
   config.headersExtendedCollectionEnabled = _config.extendedHeadersCollection.enabled
@@ -132,15 +150,17 @@ function filterExtendedHeaders (headers, excludedHeaderNames, tagPrefix, limit =
   for (const [headerName, headerValue] of Object.entries(headers)) {
     if (counter >= limit) break
     if (!excludedHeaderNames.has(headerName)) {
-      result[getHeaderTag(tagPrefix, headerName)] = String(headerValue)
+      result[getHeaderTag(tagPrefix, headerName)] = REDACTED_HEADERS.has(headerName)
+        ? '<redacted>'
+        : String(headerValue)
       counter++
     }
   }
 
   return result
 }
 
-function getCollectedHeaders (req, res, shouldCollectEventHeaders, storedResponseHeaders = {}) {
+function getCollectedHeaders (req, res, shouldCollectEventHeaders, storedResponseHeaders = {}, extendedDataCollection) {
   // Mandatory
   const mandatoryCollectedHeaders = filterHeaders(req.headers, REQUEST_HEADERS_MAP)
 
@@ -154,7 +174,8 @@ function getCollectedHeaders (req, res, shouldCollectEventHeaders, storedRespons
   const requestEventCollectedHeaders = filterHeaders(req.headers, EVENT_HEADERS_MAP)
   const responseEventCollectedHeaders = filterHeaders(responseHeaders, RESPONSE_HEADERS_MAP)
 
-  if (!config.headersExtendedCollectionEnabled || config.headersRedaction) {
+  // TODO headersExtendedCollectionEnabled and headersRedaction properties are deprecated to delete in a major
+  if ((!config.headersExtendedCollectionEnabled || config.headersRedaction) && !extendedDataCollection) {
     // Standard collection
     return Object.assign(
       mandatoryCollectedHeaders,
@@ -163,12 +184,15 @@ function getCollectedHeaders (req, res, shouldCollectEventHeaders, storedRespons
     )
   }
 
+  // TODO config.maxHeadersCollected is deprecated to delete in a major
+  const maxHeadersCollected = extendedDataCollection?.max_collected_headers ?? config.maxHeadersCollected
+
   // Extended collection
-  const requestExtendedHeadersAvailableCount =
-    config.maxHeadersCollected -
-    Object.keys(mandatoryCollectedHeaders).length -
+  const collectedHeadersCount = Object.keys(mandatoryCollectedHeaders).length +
     Object.keys(requestEventCollectedHeaders).length
 
+  const requestExtendedHeadersAvailableCount = maxHeadersCollected - collectedHeadersCount
+
   const requestEventExtendedCollectedHeaders =
     filterExtendedHeaders(
       req.headers,
@@ -178,7 +202,7 @@ function getCollectedHeaders (req, res, shouldCollectEventHeaders, storedRespons
     )
 
   const responseExtendedHeadersAvailableCount =
-    config.maxHeadersCollected -
+    maxHeadersCollected -
     Object.keys(responseEventCollectedHeaders).length
 
   const responseEventExtendedCollectedHeaders =
@@ -199,15 +223,15 @@ function getCollectedHeaders (req, res, shouldCollectEventHeaders, storedRespons
 
   // Check discarded headers
   const requestHeadersCount = Object.keys(req.headers).length
-  if (requestHeadersCount > config.maxHeadersCollected) {
+  if (requestHeadersCount > maxHeadersCollected) {
     headersTags['_dd.appsec.request.header_collection.discarded'] =
-      requestHeadersCount - config.maxHeadersCollected
+      requestHeadersCount - maxHeadersCollected
   }
 
   const responseHeadersCount = Object.keys(responseHeaders).length
-  if (responseHeadersCount > config.maxHeadersCollected) {
+  if (responseHeadersCount > maxHeadersCollected) {
     headersTags['_dd.appsec.response.header_collection.discarded'] =
-      responseHeadersCount - config.maxHeadersCollected
+      responseHeadersCount - maxHeadersCollected
   }
 
   return headersTags
@@ -307,7 +331,7 @@ function reportTruncationMetrics (rootSpan, metrics) {
   }
 }
 
-function reportAttack (attackData) {
+function reportAttack ({ events: attackData, actions }) {
   const store = storage('legacy').getStore()
   const req = store?.req
   const rootSpan = web.root(req)
@@ -338,8 +362,14 @@ function reportAttack (attackData) {
 
   rootSpan.addTags(newTags)
 
+  // TODO this should be deleted in a major
   if (config.raspBodyCollection && isRaspAttack(attackData)) {
-    reportRequestBody(rootSpan, req.body)
+    reportRequestBody(rootSpan, req.body, true)
+  }
+
+  const extendedDataCollection = actions?.extended_data_collection
+  if (extendedDataCollection) {
+    extendedDataCollectionRequest.set(req, extendedDataCollection)
   }
 }
 
@@ -398,18 +428,29 @@ function truncateRequestBody (target, depth = 0) {
   }
 }
 
-function reportRequestBody (rootSpan, requestBody) {
-  if (!requestBody) return
+function reportRequestBody (rootSpan, requestBody, comesFromRaspAction = false) {
+  if (!requestBody || Object.keys(requestBody).length === 0) return
 
   if (!rootSpan.meta_struct) {
     rootSpan.meta_struct = {}
   }
 
-  if (!rootSpan.meta_struct['http.request.body']) {
+  if (rootSpan.meta_struct['http.request.body']) {
+    // If the rasp.exceed metric exists, set also the same for the new tag
+    const currentTags = rootSpan.context()._tags
+    const sizeExceedTagValue = currentTags['_dd.appsec.rasp.request_body_size.exceeded']
+
+    if (sizeExceedTagValue) {
+      rootSpan.setTag('_dd.appsec.request_body_size.exceeded', sizeExceedTagValue)
+    }
+  } else {
     const { truncated, value } = truncateRequestBody(requestBody)
     rootSpan.meta_struct['http.request.body'] = value
     if (truncated) {
-      rootSpan.setTag('_dd.appsec.rasp.request_body_size.exceeded', 'true')
+      const sizeExceedTagKey = comesFromRaspAction
+        ? '_dd.appsec.rasp.request_body_size.exceeded' // TODO old metric to delete in a major
+        : '_dd.appsec.request_body_size.exceeded'
+      rootSpan.setTag(sizeExceedTagKey, 'true')
     }
   }
 }
@@ -496,7 +537,15 @@ function finishRequest (req, res, storedResponseHeaders) {
 
   const tags = rootSpan.context()._tags
 
-  const newTags = getCollectedHeaders(req, res, shouldCollectEventHeaders(tags), storedResponseHeaders)
+  const extendedDataCollection = extendedDataCollectionRequest.get(req)
+  const newTags = getCollectedHeaders(
+    req, res, shouldCollectEventHeaders(tags), storedResponseHeaders, extendedDataCollection
+  )
+
+  if (extendedDataCollection) {
+    // TODO add support for fastify, req.body is not available in fastify
+    reportRequestBody(rootSpan, req.body)
+  }
 
   if (tags['appsec.event'] === 'true' && typeof req.route?.path === 'string') {
     newTags['http.endpoint'] = req.route.path

packages/dd-trace/src/appsec/waf/waf_context_wrapper.js

@@ -141,7 +141,7 @@ class WAFContextWrapper {
       metrics.wafTimeout = result.timeout
 
       if (ruleTriggered) {
-        Reporter.reportAttack(result.events)
+        Reporter.reportAttack(result)
       }
 
       Reporter.reportAttributes(result.attributes)

packages/dd-trace/src/config.js

@@ -668,6 +668,7 @@ class Config {
     this._envUnprocessed['appsec.blockedTemplateJson'] = DD_APPSEC_HTTP_BLOCKED_TEMPLATE_JSON
     this._setBoolean(env, 'appsec.enabled', DD_APPSEC_ENABLED)
     this._setString(env, 'appsec.eventTracking.mode', DD_APPSEC_AUTO_USER_INSTRUMENTATION_MODE)
+    // TODO appsec.extendedHeadersCollection are deprecated, to delete in a major
     this._setBoolean(env, 'appsec.extendedHeadersCollection.enabled', DD_APPSEC_COLLECT_ALL_HEADERS)
     this._setBoolean(
       env,
@@ -679,6 +680,7 @@ class Config {
     this._setString(env, 'appsec.obfuscatorKeyRegex', DD_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP)
     this._setString(env, 'appsec.obfuscatorValueRegex', DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP)
     this._setBoolean(env, 'appsec.rasp.enabled', DD_APPSEC_RASP_ENABLED)
+    // TODO Deprecated, to delete in a major
     this._setBoolean(env, 'appsec.rasp.bodyCollection', DD_APPSEC_RASP_COLLECT_REQUEST_BODY)
     env['appsec.rateLimit'] = maybeInt(DD_APPSEC_TRACE_RATE_LIMIT)
     this._envUnprocessed['appsec.rateLimit'] = DD_APPSEC_TRACE_RATE_LIMIT

packages/dd-trace/src/config_defaults.js

@@ -36,12 +36,14 @@ module.exports = {
   'appsec.blockedTemplateJson': undefined,
   'appsec.enabled': undefined,
   'appsec.eventTracking.mode': 'identification',
+  // TODO appsec.extendedHeadersCollection is deprecated, to delete in a major
   'appsec.extendedHeadersCollection.enabled': false,
   'appsec.extendedHeadersCollection.redaction': true,
   'appsec.extendedHeadersCollection.maxHeaders': 50,
   'appsec.obfuscatorKeyRegex': defaultWafObfuscatorKeyRegex,
   'appsec.obfuscatorValueRegex': defaultWafObfuscatorValueRegex,
   'appsec.rasp.enabled': true,
+  // TODO Deprecated, to delete in a major
   'appsec.rasp.bodyCollection': false,
   'appsec.rateLimit': 100,
   'appsec.rules': undefined,

packages/dd-trace/src/remote_config/capabilities.js

@@ -31,6 +31,7 @@ module.exports = {
   ASM_RASP_CMDI: 1n << 37n,
   ASM_DD_MULTICONFIG: 1n << 42n,
   ASM_TRACE_TAGGING_RULES: 1n << 43n,
+  ASM_EXTENDED_DATA_COLLECTION: 1n << 44n,
   /*
     DO NOT ADD ARBITRARY CAPABILITIES IN YOUR CODE
     UNLESS THEY ARE ALREADY DEFINED IN THE BACKEND SOURCE OF TRUTH

packages/dd-trace/src/remote_config/index.js

@@ -96,6 +96,7 @@ function enableWafUpdate (appsecConfig) {
     rc.updateCapabilities(RemoteConfigCapabilities.ASM_HEADER_FINGERPRINT, true)
     rc.updateCapabilities(RemoteConfigCapabilities.ASM_DD_MULTICONFIG, true)
     rc.updateCapabilities(RemoteConfigCapabilities.ASM_TRACE_TAGGING_RULES, true)
+    rc.updateCapabilities(RemoteConfigCapabilities.ASM_EXTENDED_DATA_COLLECTION, true)
 
     if (appsecConfig.rasp?.enabled) {
       rc.updateCapabilities(RemoteConfigCapabilities.ASM_RASP_SQLI, true)
@@ -134,6 +135,7 @@ function disableWafUpdate () {
     rc.updateCapabilities(RemoteConfigCapabilities.ASM_HEADER_FINGERPRINT, false)
     rc.updateCapabilities(RemoteConfigCapabilities.ASM_DD_MULTICONFIG, false)
     rc.updateCapabilities(RemoteConfigCapabilities.ASM_TRACE_TAGGING_RULES, false)
+    rc.updateCapabilities(RemoteConfigCapabilities.ASM_EXTENDED_DATA_COLLECTION, false)
 
     rc.updateCapabilities(RemoteConfigCapabilities.ASM_RASP_SQLI, false)
     rc.updateCapabilities(RemoteConfigCapabilities.ASM_RASP_SSRF, false)