internal/appsec/waf: add WAF timeout metric

internal/appsec/waf.go

@@ -143,6 +143,7 @@ func newHTTPWAFEventListener(handle *waf.Handle, addresses []string, timeout tim
 			overallWAFRunDuration := time.Since(wafRunStartTime)
 
 			// Log WAF metrics.
+			op.AddTag("_dd.appsec.waf.timeouts", float64(wafCtx.TotalTimeouts()))
 			// time.Duration.Microseconds() is only as of go1.13, so we do it manually here
 			addWAFDurationTags(&op.TagsHolder, float64(wafCtx.TotalRuntime()), float64(overallWAFRunDuration.Nanoseconds()))
 			once.Do(func() {
@@ -174,8 +175,9 @@ func newGRPCWAFEventListener(handle *waf.Handle, _ []string, timeout time.Durati
 			nbEvents               uint32
 			logOnce                sync.Once
 			metricsOnce            sync.Once
-			wafRunDuration         waf.AtomicDuration
-			wafBindingsRunDuration waf.AtomicDuration
+			wafRunDuration         waf.AtomicU64
+			wafBindingsRunDuration waf.AtomicU64
+			wafTimeouts            waf.AtomicU64
 
 			events []json.RawMessage
 			mu     sync.Mutex
@@ -216,6 +218,7 @@ func newGRPCWAFEventListener(handle *waf.Handle, _ []string, timeout time.Durati
 			// callbacks, we can get rid of these variables and simply use the WAF bindings in OnHandlerOperationFinish.
 			wafBindingsRunDuration.Add(uint64(time.Since(now).Nanoseconds()))
 			wafRunDuration.Add(wafCtx.TotalRuntime())
+			wafTimeouts.Add(wafCtx.TotalTimeouts())
 			if len(event) == 0 {
 				return
 			}
@@ -226,6 +229,7 @@ func newGRPCWAFEventListener(handle *waf.Handle, _ []string, timeout time.Durati
 			mu.Unlock()
 		}))
 		op.On(grpcsec.OnHandlerOperationFinish(func(op *grpcsec.HandlerOperation, _ grpcsec.HandlerOperationRes) {
+			op.AddTag("_dd.appsec.waf.timeouts", float64(wafTimeouts))
 			addWAFDurationTags(&op.TagsHolder, float64(wafRunDuration), float64(wafBindingsRunDuration))
 			metricsOnce.Do(func() {
 				addRulesetInfoTags(&op.TagsHolder, handle.RulesetInfo())

internal/appsec/waf/waf.go

@@ -53,12 +53,17 @@ func (v *Version) String() string {
 	return fmt.Sprintf("%d.%d.%d", major, minor, patch)
 }
 
-// AtomicDuration can be used to perform atomic duration sums thanks to AtomicDuration.Add.
-type AtomicDuration uint64
+// AtomicU64 can be used to perform atomic operations on an uint64 type
+type AtomicU64 uint64
 
-// Add atomically sums the current duration value with `ns` to create a new duration value.
-func (d *AtomicDuration) Add(ns uint64) {
-	atomic.AddUint64((*uint64)(d), ns)
+// Add atomically sums the current atomic value with the provided value `v`.
+func (a *AtomicU64) Add(v uint64) {
+	atomic.AddUint64((*uint64)(a), v)
+}
+
+// Inc atomically increments the atomic value by 1
+func (a *AtomicU64) Inc() {
+	atomic.AddUint64((*uint64)(a), 1)
 }
 
 // Health allows knowing if the WAF can be used. It returns the current WAF
@@ -210,8 +215,10 @@ func (waf *Handle) Close() {
 // become available. Each request must have its own Context.
 type Context struct {
 	waf *Handle
-	// Cumulated WAF runtime for this context.
-	totalRuntimeNs AtomicDuration
+	// Cumulated WAF runtime - in nanoseconds - for this context.
+	totalRuntimeNs AtomicU64
+	// Cumulated timeout count for this context.
+	timeoutCount AtomicU64
 
 	context C.ddwaf_context
 	// Mutex protecting the use of context which is not thread-safe.
@@ -262,7 +269,12 @@ func (c *Context) runWAF(data *wafObject, timeout time.Duration) (matches []byte
 	defer C.ddwaf_result_free(&result)
 	rc := C.ddwaf_run(c.context, data.ctype(), &result, C.uint64_t(timeout/time.Microsecond))
 	c.totalRuntimeNs.Add(uint64(result.total_runtime))
-	return goReturnValues(rc, &result)
+	matches, err = goReturnValues(rc, &result)
+	if err == ErrTimeout {
+		c.timeoutCount.Inc()
+	}
+
+	return matches, err
 }
 
 // Close the WAF context by releasing its C memory and decreasing the number of
@@ -280,6 +292,11 @@ func (c *Context) TotalRuntime() uint64 {
 	return uint64(c.totalRuntimeNs)
 }
 
+// TotalTimeouts returns the cumulated amount of WAF timeotus across various run calls within the same WAF context.
+func (c *Context) TotalTimeouts() uint64 {
+	return uint64(c.timeoutCount)
+}
+
 // Translate libddwaf return values into return values suitable to a Go program.
 // Note that it is possible to have matches != nil && err != nil in case of a
 // timeout during the WAF call.

internal/appsec/waf/waf_test.go

@@ -495,6 +495,22 @@ func TestMetrics(t *testing.T) {
 		require.Greater(t, wafCtx.TotalRuntime(), uint64(0), "wafCtx runtime metric is not set")
 		require.LessOrEqual(t, wafCtx.TotalRuntime(), uint64(elapsedNS), "wafCtx runtime metric is incorrect")
 	})
+
+	t.Run("Timeouts", func(t *testing.T) {
+		wafCtx := NewContext(waf)
+		require.NotNil(t, wafCtx)
+		defer wafCtx.Close()
+		// Craft matching data to force work on the WAF
+		data := map[string]interface{}{
+			"server.request.uri.raw": "\\%uff00",
+		}
+
+		for i := uint64(1); i <= 10; i++ {
+			_, err := wafCtx.Run(data, time.Nanosecond)
+			require.Equal(t, err, ErrTimeout)
+			require.Equal(t, i, wafCtx.TotalTimeouts())
+		}
+	})
 }
 
 func requireZeroNBLiveCObjects(t testing.TB) {