chore(test): Overlay e2e test (#1579)

* make overlay store instantiation concurrent Signed-off-by: Sam Lock <sam@swlock.co.uk> * WIP: added overlay e2e test Signed-off-by: Sam Lock <sam@swlock.co.uk> * add support to compute additional env vars from ctx prior to `helmfile apply` Signed-off-by: Sam Lock <sam@swlock.co.uk> * precompute env for blob e2e test also Signed-off-by: Sam Lock <sam@swlock.co.uk> * make lint Signed-off-by: Sam Lock <sam@swlock.co.uk> --------- Signed-off-by: Sam Lock <sam@swlock.co.uk>
cerbos · May 12, 2023 · d82d66e · d82d66e
1 parent dcc269d
commit d82d66e
Show file tree

Hide file tree

Showing 12 changed files with 524 additions and 43 deletions.
diff --git a/e2e/blob/blob_test.go b/e2e/blob/blob_test.go
@@ -16,14 +16,23 @@ import (
 )
 
 func TestBlob(t *testing.T) {
-	postSetup := func(ctx e2e.Ctx) {
-		//TODO(cell) Find way to share these values with Helmfile without repeating them in both places.
+	env := make(map[string]string)
+
+	computedEnvFn := func(ctx e2e.Ctx) map[string]string {
 		minioEndpoint := fmt.Sprintf("minio-%s.%s:9000", ctx.ContextID, ctx.Namespace())
+		env["E2E_BUCKET_URL"] = blob.MinioBucketURL("cerbos", minioEndpoint)
+		env["E2E_BUCKET_PREFIX"] = "repo/"
+		env["E2E_BUCKET_USERNAME"] = "admin"
+		env["E2E_BUCKET_PASSWORD"] = "passw0rd"
+		return env
+	}
+
+	postSetup := func(ctx e2e.Ctx) {
 		p := blob.UploadParam{
-			BucketURL:    blob.MinioBucketURL("cerbos", minioEndpoint),
-			BucketPrefix: "repo/",
-			Username:     "admin",
-			Password:     "passw0rd",
+			BucketURL:    env["E2E_BUCKET_URL"],
+			BucketPrefix: env["E2E_BUCKET_PREFIX"],
+			Username:     env["E2E_BUCKET_USERNAME"],
+			Password:     env["E2E_BUCKET_PASSWORD"],
 			Directory:    filepath.Join(ctx.SourceRoot, "internal", "test", "testdata", "store"),
 		}
 
@@ -36,5 +45,5 @@ func TestBlob(t *testing.T) {
 		time.Sleep(150 * time.Millisecond)
 	}
 
-	e2e.RunSuites(t, e2e.WithContextID("blob"), e2e.WithImmutableStoreSuites(), e2e.WithPostSetup(postSetup))
+	e2e.RunSuites(t, e2e.WithContextID("blob"), e2e.WithImmutableStoreSuites(), e2e.WithPostSetup(postSetup), e2e.WithComputedEnv(computedEnvFn))
 }
diff --git a/e2e/blob/helmfile.yaml b/e2e/blob/helmfile.yaml
@@ -86,9 +86,9 @@ releases:
             mountPath: /data
       - env:
           - name: AWS_ACCESS_KEY_ID
-            value: admin
+            value: '{{ requiredEnv "E2E_BUCKET_USERNAME" }}'
           - name: AWS_SECRET_ACCESS_KEY
-            value: passw0rd
+            value: '{{ requiredEnv "E2E_BUCKET_PASSWORD" }}'
       - cerbos:
           tlsSecretName: 'cerbos-certs-{{ requiredEnv "E2E_CONTEXT_ID" }}'
           logLevel: DEBUG
@@ -118,8 +118,8 @@ releases:
             storage:
               driver: "blob"
               blob:
-                bucket: 's3://cerbos?endpoint=minio-{{ requiredEnv "E2E_CONTEXT_ID" }}.{{ requiredEnv "E2E_NS" }}.svc.cluster.local:9000&disableSSL=true&s3ForcePathStyle=true&region=local'
-                prefix: repo/
+                bucket: '{{ requiredEnv "E2E_BUCKET_URL" }}'
+                prefix: '{{ requiredEnv "E2E_BUCKET_PREFIX" }}'
                 workDir: /data
                 updatePollInterval: 0.1s
             telemetry:

diff --git a/e2e/notls/helmfile.yaml b/e2e/notls/helmfile.yaml
@@ -1,7 +1,3 @@
-repositories:
-  - name: bitnami
-    url: https://charts.bitnami.com/bitnami
-
 helmDefaults:
   cleanupOnFail: true
   wait: true

diff --git a/e2e/overlay/data.sql b/e2e/overlay/data.sql
diff --git a/e2e/overlay/helmfile.yaml b/e2e/overlay/helmfile.yaml
@@ -0,0 +1,148 @@
+repositories:
+  - name: bitnami
+    url: https://charts.bitnami.com/bitnami
+
+helmDefaults:
+  cleanupOnFail: true
+  wait: true
+  recreatePods: true
+  force: true
+  createNamespace: true
+
+releases:
+  - name: postgres
+    namespace: '{{ requiredEnv "E2E_NS" }}'
+    createNamespace: true
+    labels:
+      e2e-run: '{{ requiredEnv "E2E_RUN_ID" }}'
+      e2e-ctx: '{{ requiredEnv "E2E_CONTEXT_ID" }}'
+    chart: bitnami/postgresql
+    hooks:
+      - events: ["presync"]
+        showlogs: true
+        command: kubectl
+        args:
+          - create
+          - namespace
+          - '{{ requiredEnv "E2E_NS" }}'
+      - events: ["presync"]
+        showlogs: true
+        command: kubectl
+        args:
+          - create
+          - configmap
+          - postgres-init
+          - '--namespace={{ requiredEnv "E2E_NS" }}'
+          - '--from-file=00_init_schema.sql={{ requiredEnv "E2E_SRC_ROOT" }}/internal/storage/db/postgres/schema.sql'
+          - '--from-file=01_init_data.sql={{ requiredEnv "E2E_SRC_ROOT" }}/e2e/overlay/data.sql'
+      - events: ["postuninstall"]
+        showlogs: true
+        command: kubectl
+        args:
+          - delete
+          - configmap
+          - postgres-init
+          - '--namespace={{ requiredEnv "E2E_NS" }}'
+      - events: ["postuninstall"]
+        showlogs: true
+        command: kubectl
+        args:
+          - delete
+          - namespace
+          - '{{ requiredEnv "E2E_NS" }}'
+    values:
+      - nameOverride: '{{ requiredEnv "E2E_CONTEXT_ID" }}'
+      - auth:
+          postgresPassword: passw0rd
+      - primary:
+          initdb:
+            scriptsConfigMap: postgres-init
+          persistence:
+            enabled: false
+
+  - name: cerbos
+    namespace: '{{ requiredEnv "E2E_NS" }}'
+    needs: ["postgres"]
+    createNamespace: true
+    labels:
+      e2e-run: '{{ requiredEnv "E2E_RUN_ID" }}'
+      e2e-ctx: '{{ requiredEnv "E2E_CONTEXT_ID" }}'
+    chart: '{{ requiredEnv "E2E_SRC_ROOT" }}/deploy/charts/cerbos'
+    hooks:
+      - events: ["presync"]
+        showlogs: true
+        command: kubectl
+        args:
+          - create
+          - secret
+          - tls
+          - 'cerbos-certs-{{ requiredEnv "E2E_CONTEXT_ID" }}'
+          - '--cert={{ requiredEnv "E2E_SRC_ROOT" }}/internal/test/testdata/server/tls.crt'
+          - '--key={{ requiredEnv "E2E_SRC_ROOT" }}/internal/test/testdata/server/tls.key'
+          - '--namespace={{ requiredEnv "E2E_NS" }}'
+      - events: ["postuninstall"]
+        showlogs: true
+        command: kubectl
+        args:
+          - delete
+          - secret
+          - 'cerbos-certs-{{ requiredEnv "E2E_CONTEXT_ID" }}'
+          - '--namespace={{ requiredEnv "E2E_NS" }}'
+    values:
+      - nameOverride: '{{ requiredEnv "E2E_CONTEXT_ID" }}'
+      - image:
+          repository: '{{ env "E2E_CERBOS_IMG_REPO" | default "ghcr.io/cerbos/cerbos" }}'
+          tag: '{{ env "E2E_CERBOS_IMG_TAG" | default "dev" }}'
+      - volumes:
+        - name: cerbos-auditlog
+          emptyDir: {}
+      - volumeMounts:
+        - name: cerbos-auditlog
+          mountPath: /audit
+      - cerbos:
+          tlsSecretName: 'cerbos-certs-{{ requiredEnv "E2E_CONTEXT_ID" }}'
+          logLevel: DEBUG
+          config:
+            server:
+              playgroundEnabled: true
+              requestLimits:
+                maxActionsPerResource: 5
+                maxResourcesPerRequest: 5
+              adminAPI:
+                enabled: true
+                adminCredentials:
+                  username: cerbos
+                  passwordHash: JDJ5JDEwJC5BYjQyY2RJNG5QR2NWMmJPdnNtQU93c09RYVA0eFFGdHBrbmFEeXh1NnlIVTE1cHJNY05PCgo=
+            auxData:
+              jwt:
+                disableVerification: true
+            schema:
+              enforcement: reject
+            audit:
+              enabled: true
+              accessLogsEnabled: true
+              decisionLogsEnabled: true
+              backend: local
+              local:
+                storagePath: /audit/cerbos
+            compile:
+              # Because we're `helmfile apply`ing, our Cerbos instance persists between the two runs so we need to ensure
+              # the compilation cache is empty to ensure the second test calls out to the db
+              cacheDuration: 1ms
+            storage:
+              driver: "overlay"
+              overlay:
+                baseDriver: "postgres"
+                fallbackDriver: "git"
+                fallbackErrorThreshold: {{ requiredEnv "E2E_FALLBACK_ERR_THRESHOLD" }}
+              postgres:
+                url: {{ requiredEnv "E2E_DATABASE_URL" }}
+              git:
+                protocol: https
+                url: https://github.com/cerbos/cerbos.git
+                branch: main
+                subDir: internal/test/testdata/store
+                checkoutDir: /data
+                updatePollInterval: 60s
+            telemetry:
+              disabled: true
diff --git a/e2e/overlay/overlay_test.go b/e2e/overlay/overlay_test.go
@@ -0,0 +1,50 @@
+// Copyright 2021-2023 Zenauth Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+//go:build e2e
+
+package overlay_test
+
+import (
+	"fmt"
+	"strconv"
+	"testing"
+
+	"github.com/cerbos/cerbos/internal/test/e2e"
+	"github.com/jmoiron/sqlx"
+	"github.com/stretchr/testify/require"
+)
+
+func TestOverlay(t *testing.T) {
+	env := make(map[string]string)
+
+	computedEnvFn := func(ctx e2e.Ctx) map[string]string {
+		env["E2E_DATABASE_URL"] = fmt.Sprintf("postgres://postgres:passw0rd@%s.%s.svc.cluster.local:5432/postgres?sslmode=disable&search_path=cerbos", ctx.ContextID, ctx.Namespace())
+		env["E2E_FALLBACK_ERR_THRESHOLD"] = "1"
+		return env
+	}
+
+	// a rather hacky way simulate a DB error, by dropping tables
+	breakDB := func(ctx e2e.Ctx) {
+		db, err := sqlx.Connect("pgx", env["E2E_DATABASE_URL"])
+		require.NoError(t, err, "failed to connect to postgres")
+
+		txn := db.MustBegin()
+		txn.MustExec("DROP TABLE IF EXISTS policy CASCADE;")
+		err = txn.Commit()
+		require.NoError(t, err, "failed to drop tables from test postgres db")
+	}
+
+	t.Run("base driver success", func(t *testing.T) {
+		// TODO dedicated contextID?
+		e2e.RunSuites(t, e2e.WithContextID("postgres"), e2e.WithImmutableStoreSuites(), e2e.WithComputedEnv(computedEnvFn))
+	})
+
+	fallbackErrorThreshold, err := strconv.ParseUint(env["E2E_FALLBACK_ERR_THRESHOLD"], 10, 64)
+	require.NoError(t, err, "failed to convert fallbackErrorThreshold string to uint64")
+
+	t.Run("base driver error and fallback driver success", func(t *testing.T) {
+		// TODO dedicated contextID?
+		e2e.RunSuites(t, e2e.WithContextID("postgres"), e2e.WithImmutableStoreSuites(), e2e.WithComputedEnv(computedEnvFn), e2e.WithPostSetup(breakDB), e2e.WithOverlayMaxRetries(fallbackErrorThreshold))
+	})
+}
diff --git a/e2e/sqlite/helmfile.yaml b/e2e/sqlite/helmfile.yaml
@@ -1,7 +1,3 @@
-repositories:
-  - name: bitnami
-    url: https://charts.bitnami.com/bitnami
-
 helmDefaults:
   cleanupOnFail: true
   wait: true
@@ -59,12 +55,12 @@ releases:
       - volumes:
         - name: cerbos-auditlog
           emptyDir: {}
-        - name: cerbos-policies
+        - name: cerbos-sqlite
           emptyDir: {}
       - volumeMounts:
         - name: cerbos-auditlog
           mountPath: /audit
-        - name: cerbos-policies
+        - name: cerbos-sqlite
           mountPath: /data
       - cerbos:
           tlsSecretName: 'cerbos-certs-{{ requiredEnv "E2E_CONTEXT_ID" }}'

diff --git a/internal/server/tests.go b/internal/server/tests.go
@@ -15,6 +15,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/cenkalti/backoff/v4"
 	"github.com/google/go-cmp/cmp"
 	"github.com/stretchr/testify/require"
 	"google.golang.org/grpc"
@@ -36,6 +37,7 @@ import (
 const (
 	requestTimeout     = 5 * time.Second
 	healthPollInterval = 50 * time.Millisecond
+	retryBackoffDelay  = 5
 )
 
 type AuthCreds struct {
@@ -84,9 +86,16 @@ func readTestCase(tb testing.TB, name string, data []byte) *privatev1.ServerTest
 }
 
 type TestRunner struct {
-	Cases              []*privatev1.ServerTestCase
-	Timeout            time.Duration
-	HealthPollInterval time.Duration
+	Cases                  []*privatev1.ServerTestCase
+	Timeout                time.Duration
+	HealthPollInterval     time.Duration
+	CerbosClientMaxRetries uint64
+}
+
+// WithCerbosClientRetries is relevant to Overlay storage driver calls (specifically the e2e overlay test).
+func (tr *TestRunner) WithCerbosClientRetries(nRetries uint64) *TestRunner {
+	tr.CerbosClientMaxRetries = nRetries
+	return tr
 }
 
 func (tr *TestRunner) RunGRPCTests(addr string, opts ...grpc.DialOption) func(*testing.T) {
@@ -127,15 +136,24 @@ func (tr *TestRunner) executeGRPCTestCase(grpcConn *grpc.ClientConn, tc *private
 		case *privatev1.ServerTestCase_CheckResourceSet:
 			cerbosClient := svcv1.NewCerbosServiceClient(grpcConn)
 			want = call.CheckResourceSet.WantResponse
-			have, err = cerbosClient.CheckResourceSet(ctx, call.CheckResourceSet.Input)
+			err = backoff.Retry(func() error {
+				have, err = cerbosClient.CheckResourceSet(ctx, call.CheckResourceSet.Input)
+				return err
+			}, backoff.WithMaxRetries(backoff.NewConstantBackOff(time.Millisecond*retryBackoffDelay), tr.CerbosClientMaxRetries))
 		case *privatev1.ServerTestCase_CheckResourceBatch:
 			cerbosClient := svcv1.NewCerbosServiceClient(grpcConn)
 			want = call.CheckResourceBatch.WantResponse
-			have, err = cerbosClient.CheckResourceBatch(ctx, call.CheckResourceBatch.Input)
+			err = backoff.Retry(func() error {
+				have, err = cerbosClient.CheckResourceBatch(ctx, call.CheckResourceBatch.Input)
+				return err
+			}, backoff.WithMaxRetries(backoff.NewConstantBackOff(time.Millisecond*retryBackoffDelay), tr.CerbosClientMaxRetries))
 		case *privatev1.ServerTestCase_CheckResources:
 			cerbosClient := svcv1.NewCerbosServiceClient(grpcConn)
 			want = call.CheckResources.WantResponse
-			have, err = cerbosClient.CheckResources(ctx, call.CheckResources.Input)
+			err = backoff.Retry(func() error {
+				have, err = cerbosClient.CheckResources(ctx, call.CheckResources.Input)
+				return err
+			}, backoff.WithMaxRetries(backoff.NewConstantBackOff(time.Millisecond*retryBackoffDelay), tr.CerbosClientMaxRetries))
 		case *privatev1.ServerTestCase_PlaygroundValidate:
 			playgroundClient := svcv1.NewCerbosPlaygroundServiceClient(grpcConn)
 			want = call.PlaygroundValidate.WantResponse

diff --git a/internal/storage/overlay/store.go b/internal/storage/overlay/store.go
@@ -58,14 +58,29 @@ func NewStore(ctx context.Context, conf *Conf, confW *config.Wrapper) (*Store, e
 
 	logger := zap.S().Named(confKey+".store").With("baseDriver", conf.BaseDriver, "fallbackDriver", conf.FallbackDriver)
 
-	baseStore, err := getStore(conf.BaseDriver)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create base policy loader: %w", err)
-	}
+	p := pool.New().WithContext(ctx).WithCancelOnError().WithFirstError()
+
+	var baseStore, fallbackStore storage.Store
+	p.Go(func(ctx context.Context) error {
+		var err error
+		baseStore, err = getStore(conf.BaseDriver)
+		if err != nil {
+			return fmt.Errorf("failed to create base policy loader: %w", err)
+		}
+		return nil
+	})
 
-	fallbackStore, err := getStore(conf.FallbackDriver)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create fallback policy loader: %w", err)
+	p.Go(func(ctx context.Context) error {
+		var err error
+		fallbackStore, err = getStore(conf.FallbackDriver)
+		if err != nil {
+			return fmt.Errorf("failed to create fallback policy loader: %w", err)
+		}
+		return nil
+	})
+
+	if err := p.Wait(); err != nil {
+		return nil, err
 	}
 
 	return &Store{