feat: upgrade in-app WAF to ephemeral-capable version (#2338)

Signed-off-by: Eliott Bouhana <eliott.bouhana@datadoghq.com>
Co-authored-by: Eliott Bouhana <eliott.bouhana@datadoghq.com>
Co-authored-by: Julio Guerra <julio@datadog.com>

contrib/google.golang.org/grpc/appsec_test.go

@@ -7,6 +7,7 @@ package grpc
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"net"
 	"strings"
@@ -24,7 +25,6 @@ import (
 )
 
 func TestAppSec(t *testing.T) {
-	appsec.Start()
 	defer appsec.Stop()
 	if !appsec.Enabled() {
 		t.Skip("appsec disabled")
@@ -81,29 +81,51 @@ func TestAppSec(t *testing.T) {
 		require.Equal(t, "passed", res.Message)
 		require.NoError(t, err)
 
-		// Send a SQLi attack
-		err = stream.Send(&FixtureRequest{Name: "-1' and 1=1 union select * from users--"})
-		require.NoError(t, err)
+		for i := 0; i < 5; i++ { // Fire multiple times, each time should result in a detected event
+			// Send a SQLi attack
+			err = stream.Send(&FixtureRequest{Name: fmt.Sprintf("-%[1]d' and %[1]d=%[1]d union select * from users--", i)})
+			require.NoError(t, err)
 
-		// Check that the handler was properly called
-		res, err = stream.Recv()
-		require.Equal(t, "passed", res.Message)
-		require.NoError(t, err)
+			// Check that the handler was properly called
+			res, err = stream.Recv()
+			require.Equal(t, "passed", res.Message)
+			require.NoError(t, err)
+		}
 
 		err = stream.CloseSend()
 		require.NoError(t, err)
 		// to flush the spans
 		stream.Recv()
 
 		finished := mt.FinishedSpans()
-		require.Len(t, finished, 6)
+		require.Len(t, finished, 14)
 
 		// The request should have the attack attempts
-		event, _ := finished[5].Tag("_dd.appsec.json").(string)
-		require.NotNil(t, event)
-		require.True(t, strings.Contains(event, "crs-941-110")) // XSS attack attempt
-		require.True(t, strings.Contains(event, "crs-942-100")) // SQL-injection attack attempt
-		require.True(t, strings.Contains(event, "ua0-600-55x")) // canary rule attack attempt
+		event := finished[len(finished)-1].Tag("_dd.appsec.json")
+		require.NotNil(t, event, "the _dd.appsec.json tag was not found")
+
+		jsonText := event.(string)
+		type trigger struct {
+			Rule struct {
+				ID string `json:"id"`
+			} `json:"rule"`
+		}
+		var parsed struct {
+			Triggers []trigger `json:"triggers"`
+		}
+		err = json.Unmarshal([]byte(jsonText), &parsed)
+		require.NoError(t, err)
+
+		histogram := map[string]uint8{}
+		for _, tr := range parsed.Triggers {
+			histogram[tr.Rule.ID]++
+		}
+
+		require.EqualValues(t, 1, histogram["crs-941-110"]) // XSS attack attempt
+		require.EqualValues(t, 5, histogram["crs-942-270"]) // SQL-injection attack attempt
+		require.EqualValues(t, 1, histogram["ua0-600-55x"]) // canary rule attack attempt
+
+		require.Len(t, histogram, 3)
 	})
 }
 

go.mod

@@ -5,11 +5,11 @@ go 1.19
 require (
 	cloud.google.com/go/pubsub v1.33.0
 	github.com/99designs/gqlgen v0.17.36
-	github.com/DataDog/appsec-internal-go v1.0.1
+	github.com/DataDog/appsec-internal-go v1.0.2
 	github.com/DataDog/datadog-agent/pkg/obfuscate v0.48.0
 	github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.48.1
 	github.com/DataDog/datadog-go/v5 v5.3.0
-	github.com/DataDog/go-libddwaf v1.5.0
+	github.com/DataDog/go-libddwaf/v2 v2.1.0
 	github.com/DataDog/gostackparse v0.7.0
 	github.com/DataDog/sketches-go v1.4.2
 	github.com/IBM/sarama v1.40.0
@@ -89,7 +89,7 @@ require (
 	go.uber.org/atomic v1.11.0
 	golang.org/x/net v0.17.0
 	golang.org/x/oauth2 v0.9.0
-	golang.org/x/sys v0.13.0
+	golang.org/x/sys v0.14.0
 	golang.org/x/time v0.3.0
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2
 	google.golang.org/api v0.128.0
@@ -143,7 +143,7 @@ require (
 	github.com/eapache/go-resiliency v1.4.0 // indirect
 	github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect
 	github.com/eapache/queue v1.1.0 // indirect
-	github.com/ebitengine/purego v0.5.0-alpha.1 // indirect
+	github.com/ebitengine/purego v0.5.0 // indirect
 	github.com/elastic/elastic-transport-go/v8 v8.1.0 // indirect
 	github.com/fatih/color v1.15.0 // indirect
 	github.com/felixge/httpsnoop v1.0.3 // indirect

go.sum

@@ -624,17 +624,17 @@ github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1/go.mod h1:Vt9s
 github.com/AzureAD/microsoft-authentication-library-for-go v0.8.1/go.mod h1:4qFor3D/HDsvBME35Xy9rwW9DecL+M2sNw1ybjPtwA0=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/DataDog/appsec-internal-go v1.0.1 h1:j60HUtXEQ2uRIm8SsNnLp1Ummx/EU8iV9IFvEYmSdUM=
-github.com/DataDog/appsec-internal-go v1.0.1/go.mod h1:+Y+4klVWKPOnZx6XESG7QHydOaUGEXyH2j/vSg9JiNM=
+github.com/DataDog/appsec-internal-go v1.0.2 h1:Z+YWPlkQN+324zIk+BzKlPA1/6guKgGmYbON1/xU7gM=
+github.com/DataDog/appsec-internal-go v1.0.2/go.mod h1:+Y+4klVWKPOnZx6XESG7QHydOaUGEXyH2j/vSg9JiNM=
 github.com/DataDog/datadog-agent/pkg/obfuscate v0.48.0 h1:bUMSNsw1iofWiju9yc1f+kBd33E3hMJtq9GuU602Iy8=
 github.com/DataDog/datadog-agent/pkg/obfuscate v0.48.0/go.mod h1:HzySONXnAgSmIQfL6gOv9hWprKJkx8CicuXuUbmgWfo=
 github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.48.1 h1:5nE6N3JSs2IG3xzMthNFhXfOaXlrsdgqmJ73lndFf8c=
 github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.48.1/go.mod h1:Vc+snp0Bey4MrrJyiV2tVxxJb6BmLomPvN1RgAvjGaQ=
 github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
 github.com/DataDog/datadog-go/v5 v5.3.0 h1:2q2qjFOb3RwAZNU+ez27ZVDwErJv5/VpbBPprz7Z+s8=
 github.com/DataDog/datadog-go/v5 v5.3.0/go.mod h1:XRDJk1pTc00gm+ZDiBKsjh7oOOtJfYfglVCmFb8C2+Q=
-github.com/DataDog/go-libddwaf v1.5.0 h1:lrHP3VrEriy1M5uQuaOcKphf5GU40mBhihMAp6Ik55c=
-github.com/DataDog/go-libddwaf v1.5.0/go.mod h1:Fpnmoc2k53h6desQrH1P0/gR52CUzkLNFugE5zWwUBQ=
+github.com/DataDog/go-libddwaf/v2 v2.1.0 h1:ODQibem9zg7AC6LbU222gZwuobhnnFz5okJjh+4W3v4=
+github.com/DataDog/go-libddwaf/v2 v2.1.0/go.mod h1:X/Kc+PpP1FvvfMJvsmh/YZwGHSnhI40UkKPnDXfdTl4=
 github.com/DataDog/go-tuf v1.0.2-0.5.2 h1:EeZr937eKAWPxJ26IykAdWA4A0jQXJgkhUjqEI/w7+I=
 github.com/DataDog/go-tuf v1.0.2-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0=
 github.com/DataDog/gostackparse v0.7.0 h1:i7dLkXHvYzHV308hnkvVGDL3BR4FWl7IsXNPz/IGQh4=
@@ -1063,8 +1063,8 @@ github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4A
 github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0=
 github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
-github.com/ebitengine/purego v0.5.0-alpha.1 h1:0gVgWGb8GjKYs7cufvfNSleJAD00m2xWC26FMwOjNrw=
-github.com/ebitengine/purego v0.5.0-alpha.1/go.mod h1:ah1In8AOtksoNK6yk5z1HTJeUkC1Ez4Wk2idgGslMwQ=
+github.com/ebitengine/purego v0.5.0 h1:JrMGKfRIAM4/QVKaesIIT7m/UVjTj5GYhRSQYwfVdpo=
+github.com/ebitengine/purego v0.5.0/go.mod h1:ah1In8AOtksoNK6yk5z1HTJeUkC1Ez4Wk2idgGslMwQ=
 github.com/elastic/elastic-transport-go/v8 v8.1.0 h1:NeqEz1ty4RQz+TVbUrpSU7pZ48XkzGWQj02k5koahIE=
 github.com/elastic/elastic-transport-go/v8 v8.1.0/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI=
 github.com/elastic/go-elasticsearch/v6 v6.8.5 h1:U2HtkBseC1FNBmDr0TR2tKltL6FxoY+niDAlj5M8TK8=
@@ -2534,8 +2534,8 @@ golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
-golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
+golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=

internal/appsec/appsec.go

@@ -15,7 +15,7 @@ import (
 	"gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/dyngo"
 	"gopkg.in/DataDog/dd-trace-go.v1/internal/log"
 
-	waf "github.com/DataDog/go-libddwaf"
+	waf "github.com/DataDog/go-libddwaf/v2"
 )
 
 // Enabled returns true when AppSec is up and running. Meaning that the appsec build tag is enabled, the env var

internal/appsec/appsec_test.go

@@ -16,7 +16,7 @@ import (
 	"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"
 	"gopkg.in/DataDog/dd-trace-go.v1/internal/appsec"
 
-	waf "github.com/DataDog/go-libddwaf"
+	waf "github.com/DataDog/go-libddwaf/v2"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )

internal/appsec/dyngo/instrumentation/common.go

@@ -33,7 +33,7 @@ type (
 	// used by composition in an Operation to allow said operation to handle security events addition/retrieval.
 	// See httpsec/http.go and grpcsec/grpc.go.
 	SecurityEventsHolder struct {
-		events []json.RawMessage
+		events []any
 		mu     sync.RWMutex
 	}
 	// ContextKey is used as a key to store operations in the request's context (gRPC/HTTP)
@@ -59,14 +59,14 @@ func (m *TagsHolder) Tags() map[string]interface{} {
 
 // AddSecurityEvents adds the security events to the collected events list.
 // Thread safe.
-func (s *SecurityEventsHolder) AddSecurityEvents(events ...json.RawMessage) {
+func (s *SecurityEventsHolder) AddSecurityEvents(events []any) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	s.events = append(s.events, events...)
 }
 
 // Events returns the list of stored events.
-func (s *SecurityEventsHolder) Events() []json.RawMessage {
+func (s *SecurityEventsHolder) Events() []any {
 	s.mu.RLock()
 	defer s.mu.RUnlock()
 	return s.events
@@ -102,7 +102,7 @@ func SetAppSecEnabledTags(span TagSetter) {
 }
 
 // SetEventSpanTags sets the security event span tags into the service entry span.
-func SetEventSpanTags(span TagSetter, events []json.RawMessage) error {
+func SetEventSpanTags(span TagSetter, events []any) error {
 	// Set the appsec event span tag
 	val, err := makeEventTagValue(events)
 	if err != nil {
@@ -123,42 +123,8 @@ func SetEventSpanTags(span TagSetter, events []json.RawMessage) error {
 }
 
 // Create the value of the security event tag.
-// TODO(Julio-Guerra): a future libddwaf version should return something
-//
-//	avoiding us the following events concatenation logic which currently
-//	involves unserializing the top-level JSON arrays to concatenate them
-//	together.
-//
-// TODO(Julio-Guerra): avoid serializing the json in the request hot path
-func makeEventTagValue(events []json.RawMessage) (json.RawMessage, error) {
-	var v interface{}
-	if l := len(events); l == 1 {
-		// eventTag is the structure to use in the `_dd.appsec.json` span tag.
-		// In this case of 1 event, it already is an array as expected.
-		type eventTag struct {
-			Triggers json.RawMessage `json:"triggers"`
-		}
-		v = eventTag{Triggers: events[0]}
-	} else {
-		// eventTag is the structure to use in the `_dd.appsec.json` span tag.
-		// With more than one event, we need to concatenate the arrays together
-		// (ie. convert [][]json.RawMessage into []json.RawMessage).
-		type eventTag struct {
-			Triggers []json.RawMessage `json:"triggers"`
-		}
-		concatenated := make([]json.RawMessage, 0, l) // at least len(events)
-		for _, event := range events {
-			// Unmarshal the top level array
-			var tmp []json.RawMessage
-			if err := json.Unmarshal(event, &tmp); err != nil {
-				return nil, fmt.Errorf("unexpected error while unserializing the appsec event `%s`: %v", string(event), err)
-			}
-			concatenated = append(concatenated, tmp...)
-		}
-		v = eventTag{Triggers: concatenated}
-	}
-
-	tag, err := json.Marshal(v)
+func makeEventTagValue(events []any) (json.RawMessage, error) {
+	tag, err := json.Marshal(map[string][]any{"triggers": events})
 	if err != nil {
 		return nil, fmt.Errorf("unexpected error while serializing the appsec event span tag: %v", err)
 	}

internal/appsec/dyngo/instrumentation/grpcsec/grpc.go

@@ -11,7 +11,6 @@ package grpcsec
 
 import (
 	"context"
-	"encoding/json"
 	"reflect"
 
 	"gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/dyngo"
@@ -117,7 +116,7 @@ func StartHandlerOperation(ctx context.Context, args HandlerOperationArgs, paren
 
 // Finish the gRPC handler operation, along with the given results, and emit a
 // finish event up in the operation stack.
-func (op *HandlerOperation) Finish(res HandlerOperationRes) []json.RawMessage {
+func (op *HandlerOperation) Finish(res HandlerOperationRes) []any {
 	dyngo.FinishOperation(op, res)
 	return op.Events()
 }

internal/appsec/dyngo/instrumentation/grpcsec/grpc_test.go

@@ -7,7 +7,6 @@ package grpcsec_test
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"testing"
 
@@ -49,7 +48,7 @@ func TestUsage(t *testing.T) {
 						require.Equal(t, expectedMessage, res.Message)
 						recvFinished++
 
-						handlerOp.AddSecurityEvents(json.RawMessage(expectedMessage))
+						handlerOp.AddSecurityEvents([]any{expectedMessage})
 					}))
 				}))
 
@@ -69,7 +68,7 @@ func TestUsage(t *testing.T) {
 
 			require.Len(t, secEvents, expectedRecvOperation)
 			for i, e := range secEvents {
-				require.Equal(t, fmt.Sprintf(expectedMessageFormat, i+1), string(e))
+				require.Equal(t, fmt.Sprintf(expectedMessageFormat, i+1), e)
 			}
 		}
 	}

internal/appsec/dyngo/instrumentation/grpcsec/tags.go

@@ -6,22 +6,20 @@
 package grpcsec
 
 import (
-	"encoding/json"
-
 	"gopkg.in/DataDog/dd-trace-go.v1/ddtrace"
 	"gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/dyngo/instrumentation"
 	"gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/dyngo/instrumentation/httpsec"
 	"gopkg.in/DataDog/dd-trace-go.v1/internal/log"
 )
 
 // SetSecurityEventsTags sets the AppSec events span tags.
-func SetSecurityEventsTags(span ddtrace.Span, events []json.RawMessage) {
+func SetSecurityEventsTags(span ddtrace.Span, events []any) {
 	if err := setSecurityEventsTags(span, events); err != nil {
 		log.Error("appsec: unexpected error while creating the appsec events tags: %v", err)
 	}
 }
 
-func setSecurityEventsTags(span ddtrace.Span, events []json.RawMessage) error {
+func setSecurityEventsTags(span ddtrace.Span, events []any) error {
 	if events == nil {
 		return nil
 	}

internal/appsec/dyngo/instrumentation/grpcsec/tags_test.go

@@ -6,7 +6,6 @@
 package grpcsec
 
 import (
-	"encoding/json"
 	"fmt"
 	"net"
 	"testing"
@@ -22,7 +21,7 @@ import (
 func TestTags(t *testing.T) {
 	for _, eventCase := range []struct {
 		name          string
-		events        []json.RawMessage
+		events        []any
 		expectedTag   string
 		expectedError bool
 	}{
@@ -32,28 +31,13 @@ func TestTags(t *testing.T) {
 		},
 		{
 			name:        "one-event",
-			events:      []json.RawMessage{json.RawMessage(`["one","two"]`)},
-			expectedTag: `{"triggers":["one","two"]}`,
-		},
-		{
-			name:          "one-event-with-json-error",
-			events:        []json.RawMessage{json.RawMessage(`["one",two"]`)},
-			expectedError: true,
+			events:      []any{"one"},
+			expectedTag: `{"triggers":["one"]}`,
 		},
 		{
 			name:        "two-events",
-			events:      []json.RawMessage{json.RawMessage(`["one","two"]`), json.RawMessage(`["three","four"]`)},
-			expectedTag: `{"triggers":["one","two","three","four"]}`,
-		},
-		{
-			name:          "two-events-with-json-error",
-			events:        []json.RawMessage{json.RawMessage(`["one","two"]`), json.RawMessage(`["three,"four"]`)},
-			expectedError: true,
-		},
-		{
-			name:          "three-events-with-json-error",
-			events:        []json.RawMessage{json.RawMessage(`["one","two"]`), json.RawMessage(`["three","four"]`), json.RawMessage(`"five"`)},
-			expectedError: true,
+			events:      []any{"one", "two"},
+			expectedTag: `{"triggers":["one","two"]}`,
 		},
 	} {
 		eventCase := eventCase

internal/appsec/dyngo/instrumentation/httpsec/http.go

@@ -14,7 +14,6 @@ import (
 	"context"
 	// Blank import needed to use embed for the default blocked response payloads
 	_ "embed"
-	"encoding/json"
 	"net/http"
 	"reflect"
 	"strings"
@@ -263,7 +262,7 @@ func fromContext(ctx context.Context) *Operation {
 
 // Finish the HTTP handler operation, along with the given results and emits a
 // finish event up in the operation stack.
-func (op *Operation) Finish(res HandlerOperationRes) []json.RawMessage {
+func (op *Operation) Finish(res HandlerOperationRes) []any {
 	dyngo.FinishOperation(op, res)
 	return op.Events()
 }