test: add dedicated persistence tests (#416)

.circleci/config.yml

@@ -29,8 +29,8 @@ jobs:
         environment:
           MYSQL_ROOT_PASSWORD: test
       -
-        image: cockroachdb/cockroach:v20.1.0
-        command: start --insecure
+        image: cockroachdb/cockroach:v20.2.4
+        command: start-single-node --insecure
     working_directory: /go/src/github.com/ory/keto
     steps:
       - checkout

internal/check/engine.go

@@ -2,9 +2,11 @@ package check
 
 import (
 	"context"
+	"errors"
 	"fmt"
 
-	"github.com/ory/keto/internal/persistence"
+	"github.com/ory/herodot"
+
 	"github.com/ory/keto/internal/relationtuple"
 	"github.com/ory/keto/internal/x"
 )
@@ -71,9 +73,12 @@ func (e *Engine) checkOneIndirectionFurther(ctx context.Context, requested *rela
 	)
 
 	// loop through pages until either allowed, end of pages, or an error occurred
-	for !allowed && nextPage != persistence.PageTokenEnd && err == nil {
+	for !allowed && nextPage != x.PageTokenEnd && err == nil {
 		nextRels, nextPage, err = e.d.RelationTupleManager().GetRelationTuples(ctx, expandQuery, x.WithToken(nextPage))
-		if err != nil {
+		if errors.Is(err, herodot.ErrNotFound) {
+			allowed, err = false, nil
+			return
+		} else if err != nil {
 			return
 		}
 

internal/check/engine_test.go

@@ -382,11 +382,13 @@ func TestEngine(t *testing.T) {
 
 			// pagination assertions
 			if i >= pageSize {
-				assert.Len(t, reg.RequestedPages, 1)
+				assert.Len(t, reg.RequestedPages, 2)
 				// reset requested pages for next iteration
 				reg.RequestedPages = nil
 			} else {
-				assert.Len(t, reg.RequestedPages, 0)
+				assert.Len(t, reg.RequestedPages, 1)
+				// reset requested pages for next iteration
+				reg.RequestedPages = nil
 			}
 		}
 	})

internal/e2e/full_suit_test.go

@@ -1,17 +1,14 @@
 package e2e
 
 import (
-	"bytes"
-	"context"
 	"fmt"
 	"testing"
 
-	"github.com/stretchr/testify/assert"
+	"github.com/ory/keto/internal/x"
+
 	"github.com/stretchr/testify/require"
 
 	cliclient "github.com/ory/keto/cmd/client"
-	"github.com/ory/keto/internal/driver"
-
 	"github.com/ory/keto/internal/expand"
 
 	"github.com/ory/x/cmdx"
@@ -31,42 +28,16 @@ type (
 )
 
 func Test(t *testing.T) {
-	dsns := GetDSNs(t)
-	dsns[0].Prepare = func(ctx context.Context, t testing.TB, r driver.Registry, nn []*namespace.Namespace) {
-		// check if migrations are auto applied for dsn=memory
-		status := &bytes.Buffer{}
-		require.NoError(t, r.Migrator().MigrationStatus(ctx, status))
-		assert.Contains(t, status.String(), "Applied")
-		assert.NotContains(t, status.String(), "Pending")
-
-		// TODO
-		//nApplied := strings.Count(status.String(), "Applied")
-		//t.Cleanup(func() {
-		//	// migrate nApplied down
-		//	c.ExecNoErr(t, "migrate", "down", fmt.Sprintf("%d", nApplied))
-		//})
-
-		for _, n := range nn {
-			out := bytes.Buffer{}
-			require.NoError(t, r.NamespaceMigrator().NamespaceStatus(ctx, &out, n))
-			assert.Contains(t, out.String(), "Applied")
-			assert.NotContains(t, out.String(), "Pending")
-
-			// TODO
-			//t.Cleanup(func() {
-			//	c.ExecNoErr(t, "namespace", "migrate", "down", n.Name, "1")
-			//})
-		}
-	}
-
-	for _, dsn := range dsns {
+	for _, dsn := range x.GetDSNs(t) {
 		t.Run(fmt.Sprintf("dsn=%s", dsn.Name), func(t *testing.T) {
 			nspaces := []*namespace.Namespace{{
 				Name: "dreams",
 				ID:   0,
 			}}
 
-			ctx, reg, closeServer := startServer(t, dsn, nspaces)
+			ctx, reg := newInitializedReg(t, dsn, nspaces)
+
+			closeServer := startServer(ctx, t, reg)
 			defer closeServer()
 
 			// The test cases start here

internal/e2e/helpers.go

@@ -10,9 +10,12 @@ import (
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/assert"
+
+	"github.com/ory/keto/internal/x"
+
 	"github.com/ory/x/configx"
 	"github.com/ory/x/healthx"
-	"github.com/ory/x/sqlcon/dockertest"
 	"github.com/phayes/freeport"
 	"github.com/spf13/pflag"
 
@@ -53,68 +56,7 @@ func setup(t testing.TB) (*test.Hook, context.Context) {
 	return hook, ctx
 }
 
-func migrateEverythingUp(ctx context.Context, t testing.TB, r driver.Registry, nn []*namespace.Namespace) {
-	status := &bytes.Buffer{}
-
-	require.NoError(t, r.Migrator().MigrationStatus(ctx, status))
-
-	if strings.Contains(status.String(), "Pending") {
-		require.NoError(t, r.Migrator().MigrateUp(ctx))
-	}
-
-	for _, n := range nn {
-		require.NoError(t, r.NamespaceMigrator().MigrateNamespaceUp(ctx, n))
-	}
-
-	// TODO
-	//t.Cleanup(func() {
-	//	for _, n := range nn {
-	//		c.ExecNoErr(t, "namespace", "migrate", "down", n.Name, "1")
-	//	}
-	//
-	//	c.ExecNoErr(t, "migrate", "down", "1")
-	//})
-}
-
-type DsnT struct {
-	Name    string
-	Conn    string
-	Prepare func(context.Context, testing.TB, driver.Registry, []*namespace.Namespace)
-}
-
-func GetDSNs(t testing.TB) []*DsnT {
-	// we use a slice of structs here to always have the same execution order
-	dsns := []*DsnT{
-		{
-			Name: "memory",
-			Conn: "memory",
-		},
-	}
-	if !testing.Short() {
-		dsns = append(dsns,
-			&DsnT{
-				Name:    "mysql",
-				Conn:    dockertest.RunTestMySQL(t),
-				Prepare: migrateEverythingUp,
-			},
-			&DsnT{
-				Name:    "postgres",
-				Conn:    dockertest.RunTestPostgreSQL(t),
-				Prepare: migrateEverythingUp,
-			},
-			&DsnT{
-				Name:    "cockroach",
-				Conn:    dockertest.RunTestCockroachDB(t),
-				Prepare: migrateEverythingUp,
-			},
-		)
-	}
-	t.Cleanup(dockertest.KillAllTestDatabases)
-
-	return dsns
-}
-
-func NewInitializedReg(t testing.TB, dsn *DsnT, nspaces []*namespace.Namespace) (context.Context, driver.Registry) {
+func newInitializedReg(t testing.TB, dsn *x.DsnT, nspaces []*namespace.Namespace) (context.Context, driver.Registry) {
 	_, ctx := setup(t)
 
 	ports, err := freeport.GetFreePorts(2)
@@ -140,17 +82,51 @@ func NewInitializedReg(t testing.TB, dsn *DsnT, nspaces []*namespace.Namespace)
 	reg, err := driver.NewDefaultRegistry(ctx, flags)
 	require.NoError(t, err)
 
-	if dsn.Prepare != nil {
-		dsn.Prepare(ctx, t, reg, nspaces)
+	if dsn.Name != "memory" {
+		migrateEverythingUp(ctx, t, reg, nspaces)
 	}
+	assertMigrated(ctx, t, reg, nspaces)
 
 	return ctx, reg
 }
 
-func startServer(t testing.TB, dsn *DsnT, nspaces []*namespace.Namespace) (context.Context, driver.Registry, func()) {
-	ctx, reg := NewInitializedReg(t, dsn, nspaces)
-	// Initialization done
+func migrateEverythingUp(ctx context.Context, t testing.TB, r driver.Registry, nn []*namespace.Namespace) {
+	status := &bytes.Buffer{}
 
+	require.NoError(t, r.Migrator().MigrationStatus(ctx, status))
+
+	if strings.Contains(status.String(), "Pending") {
+		require.NoError(t, r.Migrator().MigrateUp(ctx))
+	}
+
+	for _, n := range nn {
+		require.NoError(t, r.NamespaceMigrator().MigrateNamespaceUp(ctx, n))
+	}
+
+	t.Cleanup(func() {
+		for _, n := range nn {
+			require.NoError(t, r.NamespaceMigrator().MigrateNamespaceDown(context.Background(), n, 0))
+		}
+
+		require.NoError(t, r.Migrator().MigrateDown(context.Background(), 0))
+	})
+}
+
+func assertMigrated(ctx context.Context, t testing.TB, r driver.Registry, nn []*namespace.Namespace) {
+	status := &bytes.Buffer{}
+	require.NoError(t, r.Migrator().MigrationStatus(ctx, status))
+	assert.Contains(t, status.String(), "Applied")
+	assert.NotContains(t, status.String(), "Pending")
+
+	for _, n := range nn {
+		status := &bytes.Buffer{}
+		require.NoError(t, r.NamespaceMigrator().NamespaceStatus(ctx, status, n))
+		assert.Contains(t, status.String(), "Applied")
+		assert.NotContains(t, status.String(), "Pending")
+	}
+}
+
+func startServer(ctx context.Context, t testing.TB, reg driver.Registry) func() {
 	// Start the server
 	serverCtx, serverCancel := context.WithCancel(ctx)
 	serverErr := make(chan error)
@@ -174,13 +150,11 @@ func startServer(t testing.TB, dsn *DsnT, nspaces []*namespace.Namespace) (conte
 		time.Sleep(10 * time.Millisecond)
 	}
 
-	return ctx,
-		reg,
-		// defer this close function to make sure it is shutdown on test failure as well
-		func() {
-			// stop the server
-			serverCancel()
-			// wait for it to stop
-			require.NoError(t, <-serverErr)
-		}
+	// defer this close function to make sure it is shutdown on test failure as well
+	return func() {
+		// stop the server
+		serverCancel()
+		// wait for it to stop
+		require.NoError(t, <-serverErr)
+	}
 }

internal/expand/engine.go

@@ -2,8 +2,10 @@ package expand
 
 import (
 	"context"
+	"errors"
+
+	"github.com/ory/herodot"
 
-	"github.com/ory/keto/internal/persistence"
 	"github.com/ory/keto/internal/x"
 
 	"github.com/ory/keto/internal/relationtuple"
@@ -40,7 +42,7 @@ func (e *Engine) BuildTree(ctx context.Context, subject relationtuple.Subject, r
 			rels     []*relationtuple.InternalRelationTuple
 			nextPage string
 		)
-		for nextPage != persistence.PageTokenEnd {
+		for nextPage != x.PageTokenEnd {
 			var err error
 			rels, nextPage, err = e.d.RelationTupleManager().GetRelationTuples(
 				ctx,
@@ -51,7 +53,9 @@ func (e *Engine) BuildTree(ctx context.Context, subject relationtuple.Subject, r
 				},
 				x.WithToken(nextPage),
 			)
-			if err != nil {
+			if errors.Is(err, herodot.ErrNotFound) {
+				return nil, nil
+			} else if err != nil {
 				return nil, err
 			}
 
@@ -62,10 +66,17 @@ func (e *Engine) BuildTree(ctx context.Context, subject relationtuple.Subject, r
 
 			children := make([]*Tree, len(rels))
 			for ri, r := range rels {
-				children[ri], err = e.BuildTree(ctx, r.Subject, restDepth-1)
+				child, err := e.BuildTree(ctx, r.Subject, restDepth-1)
 				if err != nil {
 					return nil, err
 				}
+				if child == nil {
+					child = &Tree{
+						Type:    Leaf,
+						Subject: r.Subject,
+					}
+				}
+				children[ri] = child
 			}
 			subTree.Children = append(subTree.Children, children...)
 		}

internal/expand/engine_test.go

@@ -254,4 +254,35 @@ func TestEngine(t *testing.T) {
 		assert.Equal(t, expectedTree, tree)
 		assert.Len(t, reg.RequestedPages, 2)
 	})
+
+	t.Run("case=handles subject sets as leaf", func(t *testing.T) {
+		reg, e := newTestEngine(t, []*namespace.Namespace{{}})
+
+		expectedTree := &expand.Tree{
+			Type: expand.Union,
+			Subject: &relationtuple.SubjectSet{
+				Object:   "root",
+				Relation: "rel",
+			},
+			Children: []*expand.Tree{
+				{
+					Type: expand.Leaf,
+					Subject: &relationtuple.SubjectSet{
+						Object:   "so",
+						Relation: "sr",
+					},
+				},
+			},
+		}
+
+		require.NoError(t, reg.WriteRelationTuples(context.Background(), &relationtuple.InternalRelationTuple{
+			Object:   expectedTree.Subject.(*relationtuple.SubjectSet).Object,
+			Relation: expectedTree.Subject.(*relationtuple.SubjectSet).Relation,
+			Subject:  expectedTree.Children[0].Subject,
+		}))
+
+		tree, err := e.BuildTree(context.Background(), expectedTree.Subject, 100)
+		require.NoError(t, err)
+		assert.Equal(t, expectedTree, tree)
+	})
 }

internal/persistence/definitions.go

@@ -32,5 +32,3 @@ var (
 	ErrNamespaceUnknown   = errors.New("namespace unknown")
 	ErrMalformedPageToken = errors.New("malformed page token")
 )
-
-const PageTokenEnd = "no other page"