feature: Audit logs (#185)

* feature: Add decision logging

* Introduce audit log

* Integrate audit log

* Add StreamAuditLogs RPC

* Add docs

* Add Kube example

* Add HTTP endpoint

* Update engine benchmark

* Add query by ID

* Cleanup context tags

* Update docs

.github/workflows/pr-test.yaml

@@ -40,7 +40,7 @@ jobs:
           fi
 
       - name: Test
-        run: make test
+        run: make test-all
 
   docs:
     name: Build docs

Makefile

@@ -38,9 +38,16 @@ generate-notice: $(GO_LICENSES)
 	@ cat hack/notice_header.txt > NOTICE.txt
 	@ $(GO_LICENSES) csv . | grep -v cerbos | sort -t ',' -k1 | column -t -N Package,URL,Licence -s ',' >> NOTICE.txt
 
+.PHONY: test-all
+test-all: test test-race
+
 .PHONY: test
 test: $(GOTESTSUM)
-	@ $(GOTESTSUM) -- -tags=tests -cover -race ./...
+	@ $(GOTESTSUM) -- -tags=tests -cover ./...
+
+.PHONY: test-race
+test-race: $(GOTESTSUM)
+	@ $(GOTESTSUM) -- -tags=tests -race ./...
 
 .PHONY: test-watch
 test-watch: $(GOTESTSUM)

NOTICE.txt

@@ -16,8 +16,13 @@ github.com/beorn7/perks/quantile                                  https://github
 github.com/bluele/gcache                                          https://github.com/bluele/gcache/blob/master/LICENSE                                             MIT
 github.com/cespare/xxhash                                         https://github.com/cespare/xxhash/blob/master/LICENSE.txt                                        MIT
 github.com/cespare/xxhash/v2                                      https://github.com/cespare/xxhash/blob/master/v2/LICENSE.txt                                     MIT
+github.com/DataDog/zstd                                           https://github.com/DataDog/zstd/blob/master/LICENSE                                              BSD-3-Clause
 github.com/davecgh/go-spew/spew                                   https://github.com/davecgh/go-spew/blob/master/spew/LICENSE                                      ISC
+github.com/dgraph-io/badger/v3                                    https://github.com/dgraph-io/badger/blob/master/v3/LICENSE                                       Apache-2.0
+github.com/dgraph-io/ristretto                                    https://github.com/dgraph-io/ristretto/blob/master/LICENSE                                       Apache-2.0
+github.com/dgraph-io/ristretto/z                                  https://github.com/dgraph-io/ristretto/blob/master/z/LICENSE                                     MIT
 github.com/doug-martin/goqu/v9                                    https://github.com/doug-martin/goqu/blob/master/v9/LICENSE                                       MIT
+github.com/dustin/go-humanize                                     https://github.com/dustin/go-humanize/blob/master/LICENSE                                        MIT
 github.com/emirpasic/gods                                         https://github.com/emirpasic/gods/blob/master/LICENSE                                            BSD-2-Clause
 github.com/envoyproxy/protoc-gen-validate/validate                https://github.com/envoyproxy/protoc-gen-validate/blob/master/validate/LICENSE                   Apache-2.0
 github.com/fatih/color                                            https://github.com/fatih/color/blob/master/LICENSE.md                                            MIT
@@ -28,9 +33,12 @@ github.com/gobwas/glob                                            https://github
 github.com/go-git/gcfg                                            https://github.com/go-git/gcfg/blob/master/LICENSE                                               BSD-3-Clause
 github.com/go-git/go-billy/v5                                     https://github.com/go-git/go-billy/blob/master/v5/LICENSE                                        Apache-2.0
 github.com/go-git/go-git/v5                                       https://github.com/go-git/go-git/blob/master/v5/LICENSE                                          Apache-2.0
+github.com/gogo/protobuf/proto                                    https://github.com/gogo/protobuf/blob/master/proto/LICENSE                                       BSD-3-Clause
 github.com/golang/groupcache/lru                                  https://github.com/golang/groupcache/blob/master/lru/LICENSE                                     Apache-2.0
 github.com/golang/protobuf                                        https://github.com/golang/protobuf/blob/master/LICENSE                                           BSD-3-Clause
+github.com/golang/snappy                                          https://github.com/golang/snappy/blob/master/LICENSE                                             BSD-3-Clause
 github.com/google/cel-go                                          https://github.com/google/cel-go/blob/master/LICENSE                                             Apache-2.0
+github.com/google/flatbuffers/go                                  https://github.com/google/flatbuffers/blob/master/go/LICENSE.txt                                 Apache-2.0
 github.com/google/go-cmp/cmp                                      https://github.com/google/go-cmp/blob/master/cmp/LICENSE                                         BSD-3-Clause
 github.com/google/gops                                            https://github.com/google/gops/blob/master/LICENSE                                               BSD-3-Clause
 github.com/gorilla/handlers                                       https://github.com/gorilla/handlers/blob/master/LICENSE                                          BSD-2-Clause
@@ -47,6 +55,7 @@ github.com/mattn/go-colorable                                     https://github
 github.com/mattn/go-isatty                                        https://github.com/mattn/go-isatty/blob/master/LICENSE                                           MIT
 github.com/matttproud/golang_protobuf_extensions/pbutil           https://github.com/matttproud/golang_protobuf_extensions/blob/master/pbutil/LICENSE              Apache-2.0
 github.com/mitchellh/go-homedir                                   https://github.com/mitchellh/go-homedir/blob/master/LICENSE                                      MIT
+github.com/oklog/ulid/v2                                          https://github.com/oklog/ulid/blob/master/v2/LICENSE                                             Apache-2.0
 github.com/OneOfOne/xxhash                                        https://github.com/OneOfOne/xxhash/blob/master/LICENSE                                           Apache-2.0
 github.com/open-policy-agent/opa                                  https://github.com/open-policy-agent/opa/blob/master/LICENSE                                     Apache-2.0
 github.com/open-policy-agent/opa/internal/jwx                     https://github.com/open-policy-agent/opa/blob/master/internal/jwx/LICENSE                        MIT

client/client_test.go

@@ -14,6 +14,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/cerbos/cerbos/client"
+	"github.com/cerbos/cerbos/internal/audit"
 	"github.com/cerbos/cerbos/internal/compile"
 	"github.com/cerbos/cerbos/internal/engine"
 	"github.com/cerbos/cerbos/internal/server"
@@ -29,10 +30,12 @@ func TestClient(t *testing.T) {
 	ctx, cancelFunc := context.WithCancel(context.Background())
 	defer cancelFunc()
 
+	auditLog := audit.NewNopLog()
+
 	store, err := disk.NewStore(ctx, &disk.Conf{Directory: dir, ScratchDir: t.TempDir()})
 	require.NoError(t, err)
 
-	eng, err := engine.New(ctx, compile.NewManager(ctx, store))
+	eng, err := engine.New(ctx, compile.NewManager(ctx, store), auditLog)
 	require.NoError(t, err)
 
 	t.Run("with_tls", func(t *testing.T) {
@@ -51,7 +54,7 @@ func TestClient(t *testing.T) {
 			ctx, cancelFunc := context.WithCancel(context.Background())
 			defer cancelFunc()
 
-			startServer(ctx, conf, store, eng)
+			startServer(ctx, conf, store, eng, auditLog)
 
 			c, err := client.New(conf.GRPCListenAddr, client.WithTLSInsecure())
 			require.NoError(t, err)
@@ -74,7 +77,7 @@ func TestClient(t *testing.T) {
 			ctx, cancelFunc := context.WithCancel(context.Background())
 			defer cancelFunc()
 
-			startServer(ctx, conf, store, eng)
+			startServer(ctx, conf, store, eng, auditLog)
 
 			c, err := client.New(conf.GRPCListenAddr, client.WithTLSInsecure())
 			require.NoError(t, err)
@@ -93,7 +96,7 @@ func TestClient(t *testing.T) {
 			ctx, cancelFunc := context.WithCancel(context.Background())
 			defer cancelFunc()
 
-			startServer(ctx, conf, store, eng)
+			startServer(ctx, conf, store, eng, auditLog)
 
 			c, err := client.New(conf.GRPCListenAddr, client.WithPlaintext())
 			require.NoError(t, err)
@@ -112,7 +115,7 @@ func TestClient(t *testing.T) {
 			ctx, cancelFunc := context.WithCancel(context.Background())
 			defer cancelFunc()
 
-			startServer(ctx, conf, store, eng)
+			startServer(ctx, conf, store, eng, auditLog)
 
 			c, err := client.New(conf.GRPCListenAddr, client.WithPlaintext())
 			require.NoError(t, err)
@@ -197,10 +200,10 @@ func getFreeListenAddr(t *testing.T) string {
 	return addr
 }
 
-func startServer(ctx context.Context, conf *server.Conf, store storage.Store, eng *engine.Engine) {
+func startServer(ctx context.Context, conf *server.Conf, store storage.Store, eng *engine.Engine, auditLog audit.Log) {
 	s := server.NewServer(conf)
 	go func() {
-		if err := s.Start(ctx, store, eng, false); err != nil {
+		if err := s.Start(ctx, store, eng, auditLog, false); err != nil {
 			panic(err)
 		}
 	}()

client/model.go

@@ -218,7 +218,7 @@ func toStructPB(v interface{}) (*structpb.Value, error) {
 	}
 
 	vv := reflect.ValueOf(v)
-	switch vv.Kind() { //nolint:exhaustive
+	switch vv.Kind() {
 	case reflect.Array, reflect.Slice:
 		arr := make([]interface{}, vv.Len())
 		for i := 0; i < vv.Len(); i++ {
@@ -239,6 +239,8 @@ func toStructPB(v interface{}) (*structpb.Value, error) {
 
 			return structpb.NewValue(m)
 		}
+	default:
+		return nil, err
 	}
 
 	return nil, err

cmd/server/server.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"os"
 	"os/signal"
+	"syscall"
 
 	"github.com/google/gops/agent"
 	"github.com/spf13/cobra"
@@ -53,6 +54,7 @@ func NewCommand() *cobra.Command {
 
 func doRun(_ *cobra.Command, _ []string) error {
 	logging.InitLogging(args.logLevel)
+	defer zap.L().Sync() //nolint:errcheck
 
 	log := zap.S().Named("server")
 
@@ -65,9 +67,10 @@ func doRun(_ *cobra.Command, _ []string) error {
 
 	if args.debugListenAddr != "" {
 		startDebugListener(args.debugListenAddr)
+		defer agent.Close()
 	}
 
-	ctx, stopFunc := signal.NotifyContext(context.Background(), os.Interrupt)
+	ctx, stopFunc := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
 	defer stopFunc()
 
 	// load any config overrides
@@ -97,7 +100,7 @@ func startDebugListener(listenAddr string) {
 
 	err := agent.Listen(agent.Options{
 		Addr:                   listenAddr,
-		ShutdownCleanup:        true,
+		ShutdownCleanup:        false,
 		ReuseSocketAddrAndPort: true,
 	})
 	if err != nil {

deploy/charts/cerbos/values-audit-log.yaml

@@ -0,0 +1,33 @@
+# Illustrates how to deploy Cerbos with an SQLite3 backend and audit logs. 
+
+cerbos:
+  config:
+    # Configure the SQLite3 storage driver
+    storage:
+      driver: "sqlite3"
+      sqlite3:
+        dsn: "file:/data/cerbos.sqlite?mode=rwc&_fk=true"  
+    # Configure audit logging
+    audit:
+      enabled: true
+      accessLogsEnabled: true
+      decisionLogsEnabled: true
+      backend: local
+      local: 
+        storagePath: /audit/cerbos
+
+# Create volumes to hold the SQLite3 database and the audit log. 
+# Note that this example uses emptyDir volumes that lose data when the pod or node is killed.
+# Use persistent volumes in production to preserve the data between pod restarts. 
+
+volumes:
+  - name: cerbos-policies
+    emptyDir: {}
+  - name: cerbos-auditlog
+    emptyDir: {}
+
+volumeMounts:
+  - name: cerbos-policies
+    mountPath: /data
+  - name: cerbos-auditlog
+    mountPath: /audit

docs/modules/api/pages/admin_api.adoc

@@ -60,3 +60,37 @@ NOTE: This endpoint requires a mutable storage driver such as xref:configuration
 ----
 {"success":{}}
 ----
+
+== List audit log entries [`/admin/auditlog/list`]
+
+When xref:configuration:audit.adoc[audit logging is enabled] you can view the audit log entries using this API endpoint.
+
+There are two kinds of audit logs:
+
+`KIND_ACCESS`:: Captured Cerbos API access logs. These records are only available if `accessLogsEnabled` is set to `true` in the  xref:configuration:audit.adoc[configuration]. 
+`KIND_DECISION`:: Decision logs captured by the engine. These records are only available if `decisionLogsEnabled` is set to `true` in the  xref:configuration:audit.adoc[configuration]. 
+
+
+You can view the last N entries (`lastN`),  entries captured between a specific time period (`between.start` and `between.end`), or a specific entry (`byCallId`). When specifying timestamps, they must be formatted as ISO 8601 timestamps.
+
+.View last 5 decision log entries
+[source,shell]
+----
+curl -k -u cerbos:cerbosAdmin \
+    'https://localhost:3592/admin/auditlog/list/KIND_DECISION?lastN=5
+----
+
+
+.View access log entries between midnight 2021-07-01 and midnight 2021-07-02
+[source,shell]
+----
+curl -k -u cerbos:cerbosAdmin \
+    'https://localhost:3592/admin/auditlog/list/KIND_ACCESS?between.start=2021-07-01T00:00:00Z&between.end=2021-07-02T00:00:00Z'
+----
+
+.View specific decision log entry
+[source,shell]
+----
+curl -k -u cerbos:cerbosAdmin \
+    'https://localhost:3592/admin/auditlog/list/KIND_DECISION?byCallId=01F9VS1N77S83MTSBBX44GYSJ6'
+----

docs/modules/configuration/nav.adoc

@@ -2,4 +2,5 @@
 * xref:engine.adoc[Engine]
 * xref:server.adoc[Server]
 * xref:storage.adoc[Storage]
+* xref:audit.adoc[Audit]
 * xref:tracing.adoc[Tracing]

docs/modules/configuration/pages/audit.adoc

@@ -0,0 +1,45 @@
+include::ROOT:partial$attributes.adoc[]
+
+= Audit block
+
+The `audit` block configures the audit logging settings for the Cerbos instance. Audit logs capture access records and decisions made by the engine along with the associated context data.
+
+
+Log storage is handled by different backends. In the free version, only the `local` backend is supported.
+
+NOTE: Audit logging has some overhead in terms of resource usage (disk IO, CPU and memory). This overhead is usually negligible unless Cerbos is running in a resource-constrained environment. If resources are scarce or if you are expecting heavy traffic, disabling audit logging might have a positive impact on performance. 
+
+
+[source,yaml,linenums]
+----
+audit: 
+  enabled: true # Set to false to completely disable audit logging.
+  accessLogsEnabled: true # Log API access attempts
+  decisionLogsEnabled: true # Log policy decisions
+  backend: local # Audit backend to use. 
+  local: # Configuration for the local audit backend
+    storagePath: /path/to/dir # Path to store the data
+    retentionPeriod: 168h # Records older than this will be automatically deleted
+----
+
+
+== Local backend
+
+The `local` backend uses an embedded key-value store to save audit records. The default settings should be sufficient for many use cases. Advanced users can fine-tune these settings using the `advanced` section.
+
+
+[source,yaml,linenums]
+----
+audit: 
+  enabled: true 
+  backend: local 
+  local: 
+    storagePath: /path/to/dir 
+    retentionPeriod: 168h 
+    advanced:
+      bufferSize: 16 # Size of the memory buffer. Increasing this will use more memory and the chances of losing data during a crash. 
+      maxBatchSize: 16 # Write batch size. If your records are small, increasing this will reduce disk IO. 
+      flushInterval: 30s # Time to keep records in memory before committing. 
+      gcInterval: 15m # How often the garbage collector runs to remove old entries from the log. 
+----
+

docs/modules/configuration/pages/index.adoc

@@ -41,14 +41,6 @@ server:
 engine: # Optional
   defaultPolicyVersion: "default" # Default policy version to assume if the request does not specify one.
 
-tracing: # Optional
-  sampleProbability: 0.1 # Sampling probability value between 0.0 and 1.0
-  exporter: jaeger # Trace exporter to use. Currently only Jaeger is supported
-  jaeger: # Required only if exporter is "jaeger"
-    serviceName: cerbos # Optional service name to report to Jaeger. Defaults to cerbos.
-    agentEndpoint: "localhost:6831" # Export to Jaeger agent. Takes precedence if both "agentEndpoint" and "collectorEndpoint" are defined. 
-    collectorEndpoint: "http://localhost:14268" # Export to Jaeger collector.
-
 storage:
   driver: "disk" # Valid values are "disk", "git" or "sqlite3"
   disk: # Only required if "driver" is "disk"
@@ -71,6 +63,23 @@ storage:
       privateKeyFile: ${HOME}/.ssh/id_rsa
   sqlite3: # Only required if the "driver" is "sqlite3"
     dsn: ":memory:?_fk=true"
+
+tracing: # Optional
+  sampleProbability: 0.1 # Sampling probability value between 0.0 and 1.0
+  exporter: jaeger # Trace exporter to use. Currently only Jaeger is supported
+  jaeger: # Required only if exporter is "jaeger"
+    serviceName: cerbos # Optional service name to report to Jaeger. Defaults to cerbos.
+    agentEndpoint: "localhost:6831" # Export to Jaeger agent. Takes precedence if both "agentEndpoint" and "collectorEndpoint" are defined. 
+    collectorEndpoint: "http://localhost:14268" # Export to Jaeger collector.
+
+audit: # Optional
+  enabled: false # Enable audit logging
+  accessLogsEnabled: true # Log API access attempts
+  decisionLogsEnabled: true # Log policy decisions
+  backend: local # Audit backend to use. 
+  local: # Configuration for the local audit backend
+    storagePath: /path/to/dir # Path to store the data
+    retentionPeriod: 168h # How long to keep records for
 ----
 
 

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/bluele/gcache v0.0.2
 	github.com/bufbuild/buf v0.43.2
 	github.com/cespare/xxhash v1.1.0
+	github.com/dgraph-io/badger/v3 v3.2103.0
 	github.com/doug-martin/goqu/v9 v9.13.0
 	github.com/envoyproxy/protoc-gen-validate v0.6.1
 	github.com/fatih/color v1.12.0
@@ -32,6 +33,7 @@ require (
 	github.com/kavu/go_reuseport v1.5.0
 	github.com/lyft/protoc-gen-star v0.5.3 // indirect
 	github.com/mattn/go-isatty v0.0.13
+	github.com/oklog/ulid/v2 v2.0.2
 	github.com/open-policy-agent/opa v0.29.4
 	github.com/planetscale/vtprotobuf v0.0.0-20210616093554-9236f7c7b8ca
 	github.com/prometheus/client_golang v1.11.0