From 324d2724d6c1d462c9bbb7b5c1f3a676cf210ad3 Mon Sep 17 00:00:00 2001 From: Valentin Zakharov Date: Tue, 11 Jun 2024 17:15:19 +0200 Subject: [PATCH] Collect and report RASP events including Stack traces --- .../appsec/gateway/AppSecRequestContext.java | 48 ++++++-- .../datadog/appsec/gateway/GatewayBridge.java | 10 ++ .../appsec/powerwaf/PowerWAFModule.java | 35 ++++++ .../appsec/powerwaf/PowerWAFResultData.java | 11 +- .../datadog/appsec/report/AppSecEvent.java | 19 ++- .../stack_trace/StackTraceCollection.java | 12 ++ .../appsec/stack_trace/StackTraceEvent.java | 35 ++++++ .../datadog/appsec/util/ObjectFlattener.java | 26 ++++ .../util/ObjectFlattenerSpecification.groovy | 111 ++++++++++++++++++ 9 files changed, 296 insertions(+), 11 deletions(-) create mode 100644 dd-java-agent/appsec/src/main/java/com/datadog/appsec/stack_trace/StackTraceCollection.java create mode 100644 dd-java-agent/appsec/src/main/java/com/datadog/appsec/stack_trace/StackTraceEvent.java create mode 100644 dd-java-agent/appsec/src/main/java/com/datadog/appsec/util/ObjectFlattener.java create mode 100644 dd-java-agent/appsec/src/test/groovy/com/datadog/appsec/util/ObjectFlattenerSpecification.groovy diff --git a/dd-java-agent/appsec/src/main/java/com/datadog/appsec/gateway/AppSecRequestContext.java b/dd-java-agent/appsec/src/main/java/com/datadog/appsec/gateway/AppSecRequestContext.java index 07ffe201fd5..7e69858f5af 100644 --- a/dd-java-agent/appsec/src/main/java/com/datadog/appsec/gateway/AppSecRequestContext.java +++ b/dd-java-agent/appsec/src/main/java/com/datadog/appsec/gateway/AppSecRequestContext.java @@ -3,7 +3,10 @@ import com.datadog.appsec.event.data.Address; import com.datadog.appsec.event.data.DataBundle; import com.datadog.appsec.report.AppSecEvent; +import com.datadog.appsec.stack_trace.StackTraceCollection; +import com.datadog.appsec.stack_trace.StackTraceEvent; import com.datadog.appsec.util.StandardizedLogging; +import datadog.trace.api.Config; import datadog.trace.api.http.StoredBodySupplier; import datadog.trace.api.internal.TraceSegment; import io.sqreen.powerwaf.Additive; @@ -72,7 +75,8 @@ public class AppSecRequestContext implements DataBundle, Closeable { } private final ConcurrentHashMap, Object> persistentData = new ConcurrentHashMap<>(); - private Collection collectedEvents; // guarded by this + private Collection appSecEvents; // guarded by this + private Collection stackTraceEvents; // guarded by this // assume these will always be written and read by the same thread private String scheme; @@ -405,27 +409,42 @@ public CharSequence getStoredRequestBody() { return storedRequestBodySupplier.get(); } - public void reportEvents(Collection events) { - for (AppSecEvent event : events) { + public void reportEvents(Collection appSecEvents) { + for (AppSecEvent event : appSecEvents) { StandardizedLogging.attackDetected(log, event); } synchronized (this) { - if (this.collectedEvents == null) { - this.collectedEvents = new ArrayList<>(); + if (this.appSecEvents == null) { + this.appSecEvents = new ArrayList<>(); } try { - this.collectedEvents.addAll(events); + this.appSecEvents.addAll(appSecEvents); } catch (UnsupportedOperationException e) { throw new IllegalStateException("Events cannot be added anymore"); } } } + public void reportStackTrace(StackTraceEvent stackTraceEvent) { + synchronized (this) { + if (this.stackTraceEvents == null) { + this.stackTraceEvents = new ArrayList<>(); + } + try { + if (stackTraceEvents.size() <= Config.get().getAppSecMaxStackTraces()) { + this.stackTraceEvents.add(stackTraceEvent); + } + } catch (UnsupportedOperationException e) { + throw new IllegalStateException("Stacktrace cannot be added anymore"); + } + } + } + Collection transferCollectedEvents() { Collection events; synchronized (this) { - events = this.collectedEvents; - this.collectedEvents = Collections.emptyList(); + events = this.appSecEvents; + this.appSecEvents = Collections.emptyList(); } if (events != null) { return events; @@ -434,6 +453,19 @@ Collection transferCollectedEvents() { } } + StackTraceCollection transferStackTracesCollection() { + Collection stackTraces; + synchronized (this) { + stackTraces = this.stackTraceEvents; + this.stackTraceEvents = Collections.emptyList(); + } + if (stackTraces != null) { + return new StackTraceCollection(stackTraces); + } else { + return null; + } + } + public void reportApiSchemas(Map schemas) { if (schemas == null || schemas.isEmpty()) return; diff --git a/dd-java-agent/appsec/src/main/java/com/datadog/appsec/gateway/GatewayBridge.java b/dd-java-agent/appsec/src/main/java/com/datadog/appsec/gateway/GatewayBridge.java index 837b96f6628..2e152a5c8eb 100644 --- a/dd-java-agent/appsec/src/main/java/com/datadog/appsec/gateway/GatewayBridge.java +++ b/dd-java-agent/appsec/src/main/java/com/datadog/appsec/gateway/GatewayBridge.java @@ -19,6 +19,8 @@ import com.datadog.appsec.event.data.SingletonDataBundle; import com.datadog.appsec.report.AppSecEvent; import com.datadog.appsec.report.AppSecEventWrapper; +import com.datadog.appsec.stack_trace.StackTraceCollection; +import com.datadog.appsec.util.ObjectFlattener; import datadog.trace.api.Config; import datadog.trace.api.DDTags; import datadog.trace.api.function.TriConsumer; @@ -162,6 +164,14 @@ public void init() { writeRequestHeaders(traceSeg, REQUEST_HEADERS_ALLOW_LIST, ctx.getRequestHeaders()); writeResponseHeaders( traceSeg, RESPONSE_HEADERS_ALLOW_LIST, ctx.getResponseHeaders()); + + // Report collected stack traces + StackTraceCollection stackTraceCollection = ctx.transferStackTracesCollection(); + if (stackTraceCollection != null) { + Object flatStruct = ObjectFlattener.flatten(stackTraceCollection); + traceSeg.setMetaStructTop("_dd.stack", flatStruct); + } + } else if (hasUserTrackingEvent(traceSeg)) { // Report all collected request headers on user tracking event writeRequestHeaders(traceSeg, REQUEST_HEADERS_ALLOW_LIST, ctx.getRequestHeaders()); diff --git a/dd-java-agent/appsec/src/main/java/com/datadog/appsec/powerwaf/PowerWAFModule.java b/dd-java-agent/appsec/src/main/java/com/datadog/appsec/powerwaf/PowerWAFModule.java index 67f460b02f2..1d0effce578 100644 --- a/dd-java-agent/appsec/src/main/java/com/datadog/appsec/powerwaf/PowerWAFModule.java +++ b/dd-java-agent/appsec/src/main/java/com/datadog/appsec/powerwaf/PowerWAFModule.java @@ -14,6 +14,8 @@ import com.datadog.appsec.event.data.KnownAddresses; import com.datadog.appsec.gateway.AppSecRequestContext; import com.datadog.appsec.report.AppSecEvent; +import com.datadog.appsec.stack_trace.StackTraceEvent; +import com.datadog.appsec.stack_trace.StackTraceEvent.Frame; import com.datadog.appsec.util.StandardizedLogging; import com.squareup.moshi.JsonAdapter; import com.squareup.moshi.Moshi; @@ -26,6 +28,7 @@ import datadog.trace.api.telemetry.WafMetricCollector; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import datadog.trace.bootstrap.instrumentation.api.AgentTracer; +import datadog.trace.util.stacktrace.StackWalkerFactory; import io.sqreen.powerwaf.Additive; import io.sqreen.powerwaf.Powerwaf; import io.sqreen.powerwaf.PowerwafConfig; @@ -57,6 +60,7 @@ import java.util.UUID; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; +import java.util.stream.IntStream; import javax.annotation.Nonnull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -76,6 +80,8 @@ public class PowerWAFModule implements AppSecModule { private static final Map DEFAULT_ACTIONS; + private static final String EXPLOIT_DETECTED_MSG = "Exploit detected"; + private static class ActionInfo { final String type; final Map parameters; @@ -430,6 +436,11 @@ public void onDataAvailable( Flow.Action.RequestBlockingAction rba = createRedirectRequestAction(actionInfo); flow.setAction(rba); break; + } else if ("generate_stack".equals(actionInfo.type) + && Config.get().isAppSecStackTraceEnabled()) { + String stackId = (String) actionInfo.parameters.get("stack_id"); + StackTraceEvent stackTraceEvent = createExploitStackTraceEvent(stackId); + reqCtx.reportStackTrace(stackTraceEvent); } else { log.info("Ignoring action with type {}", actionInfo.type); } @@ -497,6 +508,29 @@ private Flow.Action.RequestBlockingAction createRedirectRequestAction(ActionInfo } } + private StackTraceEvent createExploitStackTraceEvent(String stackId) { + if (stackId == null || stackId.isEmpty()) { + return null; + } + List result = generateUserCodeStackTrace(); + return new StackTraceEvent(stackId, EXPLOIT_DETECTED_MSG, result); + } + + /** Function generates stack trace of the user code (excluding datadog classes) */ + private List generateUserCodeStackTrace() { + int stackCapacity = Config.get().getAppSecMaxStackTraceDepth(); + List elements = + StackWalkerFactory.INSTANCE.walk( + stream -> + stream + .filter(elem -> !elem.getClassName().startsWith("com.datadog")) + .limit(stackCapacity) + .collect(Collectors.toList())); + return IntStream.range(0, elements.size()) + .mapToObj(idx -> new Frame(elements.get(idx), idx)) + .collect(Collectors.toList()); + } + private Powerwaf.ResultWithData doRunPowerwaf( AppSecRequestContext reqCtx, DataBundle newData, @@ -563,6 +597,7 @@ private AppSecEvent buildEvent(PowerWAFResultData wafResult) { .withRule(wafResult.rule) .withRuleMatches(wafResult.rule_matches) .withSpanId(spanId) + .withStackId(wafResult.stack_id) .build(); } diff --git a/dd-java-agent/appsec/src/main/java/com/datadog/appsec/powerwaf/PowerWAFResultData.java b/dd-java-agent/appsec/src/main/java/com/datadog/appsec/powerwaf/PowerWAFResultData.java index c7043214319..4426514e4ad 100644 --- a/dd-java-agent/appsec/src/main/java/com/datadog/appsec/powerwaf/PowerWAFResultData.java +++ b/dd-java-agent/appsec/src/main/java/com/datadog/appsec/powerwaf/PowerWAFResultData.java @@ -6,6 +6,7 @@ public class PowerWAFResultData { Rule rule; List rule_matches; + String stack_id; public static class RuleMatch { String operator; @@ -19,10 +20,16 @@ public static class Rule { Map tags; } - public static class Parameter { + public static class Parameter extends MatchInfo { + MatchInfo resources; + MatchInfo params; + MatchInfo db_type; + List highlight; + } + + public static class MatchInfo { String address; List key_path; String value; - List highlight; } } diff --git a/dd-java-agent/appsec/src/main/java/com/datadog/appsec/report/AppSecEvent.java b/dd-java-agent/appsec/src/main/java/com/datadog/appsec/report/AppSecEvent.java index d1d3b0f88b7..3c5c8219690 100644 --- a/dd-java-agent/appsec/src/main/java/com/datadog/appsec/report/AppSecEvent.java +++ b/dd-java-agent/appsec/src/main/java/com/datadog/appsec/report/AppSecEvent.java @@ -18,6 +18,9 @@ public class AppSecEvent { @com.squareup.moshi.Json(name = "span_id") private Long spanId; + @com.squareup.moshi.Json(name = "stack_id") + private String stackId; + public Rule getRule() { return rule; } @@ -30,6 +33,10 @@ public Long getSpanId() { return spanId; } + public String getStackId() { + return stackId; + } + @Override public String toString() { StringBuilder sb = new StringBuilder(); @@ -48,6 +55,9 @@ public String toString() { sb.append("spanId"); sb.append('='); sb.append(((this.spanId == null) ? "" : this.spanId)); + sb.append("stackId"); + sb.append('='); + sb.append(((this.stackId == null) ? "" : this.stackId)); if (sb.charAt((sb.length() - 1)) == ',') { sb.setCharAt((sb.length() - 1), ']'); } else { @@ -62,6 +72,7 @@ public int hashCode() { result = ((result * 31) + ((this.rule == null) ? 0 : this.rule.hashCode())); result = ((result * 31) + ((this.ruleMatches == null) ? 0 : this.ruleMatches.hashCode())); result = ((result * 31) + ((this.spanId == null) ? 0 : this.spanId.hashCode())); + result = ((result * 31) + ((this.stackId == null) ? 0 : this.stackId.hashCode())); return result; } @@ -76,7 +87,8 @@ public boolean equals(Object other) { AppSecEvent rhs = ((AppSecEvent) other); return ((Objects.equals(this.rule, rhs.rule)) && (Objects.equals(this.ruleMatches, rhs.ruleMatches)) - && (Objects.equals(this.spanId, rhs.spanId))); + && (Objects.equals(this.spanId, rhs.spanId)) + && (Objects.equals(this.stackId, rhs.stackId))); } public static class Builder { @@ -108,5 +120,10 @@ public Builder withSpanId(Long spanId) { this.instance.spanId = spanId; return this; } + + public Builder withStackId(String stackId) { + this.instance.stackId = stackId; + return this; + } } } diff --git a/dd-java-agent/appsec/src/main/java/com/datadog/appsec/stack_trace/StackTraceCollection.java b/dd-java-agent/appsec/src/main/java/com/datadog/appsec/stack_trace/StackTraceCollection.java new file mode 100644 index 00000000000..27aec7249bb --- /dev/null +++ b/dd-java-agent/appsec/src/main/java/com/datadog/appsec/stack_trace/StackTraceCollection.java @@ -0,0 +1,12 @@ +package com.datadog.appsec.stack_trace; + +import java.util.Collection; + +public class StackTraceCollection { + public final Collection exploit; + // TODO: Add vulnerability and exception collections for future use in IAST and APM + + public StackTraceCollection(Collection exploit) { + this.exploit = exploit; + } +} diff --git a/dd-java-agent/appsec/src/main/java/com/datadog/appsec/stack_trace/StackTraceEvent.java b/dd-java-agent/appsec/src/main/java/com/datadog/appsec/stack_trace/StackTraceEvent.java new file mode 100644 index 00000000000..44ff91b2392 --- /dev/null +++ b/dd-java-agent/appsec/src/main/java/com/datadog/appsec/stack_trace/StackTraceEvent.java @@ -0,0 +1,35 @@ +package com.datadog.appsec.stack_trace; + +import java.util.List; + +public class StackTraceEvent { + + public final String id; + public final String language = "java"; + public final String message; + public final List frames; + + public StackTraceEvent(String id, String message, List frames) { + this.id = id; + this.message = message; + this.frames = frames; + } + + public static class Frame { + public final int id; + public final String text; + public final String file; + public final int line; + public final String class_name; + public final String function; + + public Frame(StackTraceElement element, int id) { + this.id = id; + this.text = element.toString(); + this.file = element.getFileName(); + this.line = element.getLineNumber(); + this.class_name = element.getClassName(); + this.function = element.getMethodName(); + } + } +} diff --git a/dd-java-agent/appsec/src/main/java/com/datadog/appsec/util/ObjectFlattener.java b/dd-java-agent/appsec/src/main/java/com/datadog/appsec/util/ObjectFlattener.java new file mode 100644 index 00000000000..f43ecfafae3 --- /dev/null +++ b/dd-java-agent/appsec/src/main/java/com/datadog/appsec/util/ObjectFlattener.java @@ -0,0 +1,26 @@ +package com.datadog.appsec.util; + +import com.squareup.moshi.JsonAdapter; +import com.squareup.moshi.Moshi; + +/** + * This class provides a utility method to flatten an object into a Map. The object's fields are + * used as keys in the Map, and the field values are used as values. The field values are + * recursively flattened if they are custom objects. + */ +public class ObjectFlattener { + + private static final JsonAdapter JSON_ADAPTER = + new Moshi.Builder().build().adapter(Object.class); + + /** + * Flattens an object into a Map. + * + * @param obj the object to flatten + * @return the flattened object as a Map, or the original object if it's a primitive type or a + * Collection or a Map. Returns null if the input object is null. + */ + public static Object flatten(Object obj) { + return JSON_ADAPTER.toJsonValue(obj); + } +} diff --git a/dd-java-agent/appsec/src/test/groovy/com/datadog/appsec/util/ObjectFlattenerSpecification.groovy b/dd-java-agent/appsec/src/test/groovy/com/datadog/appsec/util/ObjectFlattenerSpecification.groovy new file mode 100644 index 00000000000..c12bcb592d8 --- /dev/null +++ b/dd-java-agent/appsec/src/test/groovy/com/datadog/appsec/util/ObjectFlattenerSpecification.groovy @@ -0,0 +1,111 @@ +package com.datadog.appsec.util + +import datadog.trace.test.util.DDSpecification + +class ObjectFlattenerSpecification extends DDSpecification { + + def "flatten should return null for null input"() { + expect: + ObjectFlattener.flatten(null) == null + } + + def "flatten should return primitive types as is"() { + expect: + ObjectFlattener.flatten(42) == 42 + ObjectFlattener.flatten("test") == "test" + ObjectFlattener.flatten(true) == true + // ObjectFlattener.flatten(3.14) == 3.14 + } + + def "flatten should return collections with flattened elements"() { + given: + def list = [1, "test", [nested: "value"]] as List + + when: + def result = ObjectFlattener.flatten(list) + + then: + result instanceof List + result.size() == 3 + result[0] == 1 + result[1] == "test" + result[2] == [nested: "value"] + } + + def "flatten should return maps with flattened values"() { + given: + def map = [key1: 1, key2: "test", key3: [nested: "value"]] as Map + + when: + def result = ObjectFlattener.flatten(map) + + then: + result instanceof Map + result.size() == 3 + result.key1 == 1 + result.key2 == "test" + result.key3 == [nested: "value"] + } + + def "flatten should return custom objects as map"() { + given: + def nestedObject = new NestedObject("NestedName", 456) + def testObject = new TestObject(name: "TestName", value: 123, nestedObject: nestedObject) + + when: + def result = ObjectFlattener.flatten(testObject) + + then: + result instanceof Map + result.name == "TestName" + result.value == 123 + result.nestedObject instanceof Map + result.nestedObject.nestedName == "NestedName" + result.nestedObject.nestedValue == 456 + } + + def "flatten should handle nested collections and maps"() { + given: + def nestedMap = [key1: [nestedKey: "nestedValue"]] as Map + def nestedList = [1, [2, 3], ["nested": "value"]] as List + def testObject = new TestObject(name: "TestName", value: 123, nestedObject: new NestedObject("NestedName", 456)) + testObject.setList(nestedList) + testObject.setMap(nestedMap) + + when: + def result = ObjectFlattener.flatten(testObject) + + then: + result instanceof Map + result.name == "TestName" + result.value == 123 + result.nestedObject instanceof Map + result.nestedObject.nestedName == "NestedName" + result.nestedObject.nestedValue == 456 + result.list instanceof List + result.list.size() == 3 + result.list[0] == 1 + result.list[1] == [2, 3] + result.list[2] == ["nested": "value"] + result.map instanceof Map + result.map.key1 == [nestedKey: "nestedValue"] + } + + private static class TestObject { + String name + int value + NestedObject nestedObject + List list + Map map + } + + private static class NestedObject { + String nestedName + int nestedValue + + NestedObject(String nestedName, int nestedValue) { + this.nestedName = nestedName + this.nestedValue = nestedValue + } + } +}