refactor: persistence table structure

.bin/go.mod

@@ -4,8 +4,6 @@ go 1.16
 
 replace github.com/goreleaser/nfpm => github.com/goreleaser/nfpm v1.10.2
 
-replace github.com/ory/kratos-client-go => github.com/ory/kratos-client-go v0.5.4-alpha.1.0.20210210170256-960b093d8bf9
-
 replace github.com/ory/kratos/corp => github.com/ory/kratos/corp v0.0.0-20210118092700-c2358be1e867
 
 replace github.com/oleiade/reflections => github.com/oleiade/reflections v1.0.1
@@ -15,7 +13,6 @@ require (
 	github.com/go-swagger/go-swagger v0.26.1
 	github.com/goreleaser/godownloader v0.1.1-0.20200426152203-fd8ad8f7dd78
 	github.com/mattn/goveralls v0.0.7
-	github.com/ory/cli v0.0.49
 	github.com/ory/go-acc v0.2.6
 	github.com/pseudomuto/protoc-gen-doc v1.4.1
 	golang.org/x/tools v0.1.0

.bin/indirect_pins.go

@@ -7,7 +7,6 @@ import (
 	_ "github.com/mattn/goveralls"
 	_ "golang.org/x/tools/cmd/goimports"
 
-	_ "github.com/ory/cli"
 	_ "github.com/ory/go-acc"
 
 	// Protobuf and gRPC related tools

.gitignore

@@ -2,3 +2,4 @@ node_modules/
 .idea/
 coverage.txt
 dist/
+**/*.sqlite

Makefile

@@ -6,7 +6,6 @@ export PWD := $(shell pwd)
 GO_DEPENDENCIES = github.com/go-swagger/go-swagger/cmd/swagger \
 				  golang.org/x/tools/cmd/goimports \
 				  github.com/mattn/goveralls \
-				  github.com/ory/cli \
 				  github.com/ory/go-acc \
 				  github.com/bufbuild/buf/cmd/buf \
 				  google.golang.org/protobuf/cmd/protoc-gen-go \
@@ -16,12 +15,16 @@ GO_DEPENDENCIES = github.com/go-swagger/go-swagger/cmd/swagger \
 
 define make-go-dependency
   # go install is responsible for not re-building when the code hasn't changed
-  .bin/$(notdir $1): go.mod go.sum Makefile
+  .bin/$(notdir $1): .bin/go.mod .bin/go.sum Makefile
 		cd .bin; GOBIN=$(PWD)/.bin/ go install $1
 endef
 $(foreach dep, $(GO_DEPENDENCIES), $(eval $(call make-go-dependency, $(dep))))
 $(call make-lint-dependency)
 
+.bin/ory: Makefile
+		bash <(curl https://raw.githubusercontent.com/ory/cli/master/install.sh) -b .bin v0.0.57
+		touch -a -m .bin/ory
+
 node_modules: package.json package-lock.json Makefile
 		npm ci
 
@@ -95,3 +98,11 @@ test-docs-samples:
 		npm i \
 		&& \
 		npm test
+
+.PHONY: migrations-render
+migrations-render: .bin/ory
+		ory dev pop migration render internal/persistence/sql/migrations/templates internal/persistence/sql/migrations/sql
+
+.PHONY: migrations-render-replace
+migrations-render-replace: .bin/ory
+		ory dev pop migration render -r internal/persistence/sql/migrations/templates internal/persistence/sql/migrations/sql

cmd/client/test_helpers.go

@@ -4,6 +4,8 @@ import (
 	"net"
 	"testing"
 
+	"github.com/ory/keto/internal/driver/config"
+
 	"github.com/ory/x/cmdx"
 	"github.com/spf13/cobra"
 	"github.com/stretchr/testify/require"
@@ -35,8 +37,9 @@ const (
 
 func NewTestServer(t *testing.T, rw ServerType, nspaces []*namespace.Namespace, newCmd func() *cobra.Command) *TestServer {
 	ts := &TestServer{
-		Reg: driver.NewMemoryTestRegistry(t, nspaces),
+		Reg: driver.NewSqliteTestRegistry(t, false),
 	}
+	require.NoError(t, ts.Reg.Config().Set(config.KeyNamespaces, nspaces))
 
 	switch rw {
 	case ReadServer:

cmd/migrate/down.go

@@ -29,12 +29,12 @@ func newDownCmd() *cobra.Command {
 				return fmt.Errorf("malformed argument %s for <steps>: %+v", args[0], err)
 			}
 
-			reg, err := driver.NewDefaultRegistry(cmd.Context(), cmd.Flags())
+			reg, err := driver.NewDefaultRegistry(cmd.Context(), cmd.Flags(), true)
 			if err != nil {
 				return err
 			}
 
-			mb, err := reg.Migrator().MigrationBox(cmd.Context())
+			mb, err := reg.MigrationBox()
 			if err != nil {
 				return err
 			}

cmd/migrate/migrate_test.go

@@ -3,10 +3,13 @@ package migrate
 import (
 	"bytes"
 	"context"
-	"regexp"
 	"strings"
 	"testing"
 
+	"github.com/ory/x/dbal"
+
+	"github.com/ory/keto/internal/x/dbx"
+
 	"github.com/ory/x/cmdx"
 	"github.com/ory/x/configx"
 	"github.com/sirupsen/logrus/hooks/test"
@@ -17,7 +20,6 @@ import (
 	"github.com/ory/keto/internal/driver"
 	"github.com/ory/keto/internal/driver/config"
 	"github.com/ory/keto/internal/namespace"
-	"github.com/ory/keto/internal/x"
 )
 
 func assertAllApplied(t *testing.T, status string) {
@@ -54,14 +56,14 @@ func TestMigrate(t *testing.T) {
 		}
 	}
 
-	for _, dsn := range x.GetDSNs(t) {
-		if dsn.Name == "memory" {
+	for _, dsn := range dbx.GetDSNs(t, false) {
+		if dbal.IsMemorySQLite(dsn.Conn) {
 			t.Run("dsn=memory", func(t *testing.T) {
 				t.Run("case=auto migrates", func(t *testing.T) {
 					hook := &test.Hook{}
 					ctx := context.WithValue(context.Background(), driver.LogrusHookContextKey, hook)
 
-					cf := x.ConfigFile(t, map[string]interface{}{
+					cf := dbx.ConfigFile(t, map[string]interface{}{
 						config.KeyDSN:        dsn.Conn,
 						config.KeyNamespaces: nspaces,
 						"log.level":          "debug",
@@ -78,7 +80,7 @@ func TestMigrate(t *testing.T) {
 				hook := &test.Hook{}
 				ctx := context.WithValue(context.Background(), driver.LogrusHookContextKey, hook)
 
-				cf := x.ConfigFile(t, map[string]interface{}{
+				cf := dbx.ConfigFile(t, map[string]interface{}{
 					config.KeyDSN:        dsn.Conn,
 					config.KeyNamespaces: nspaces,
 					"log.level":          "debug",
@@ -125,25 +127,6 @@ func TestMigrate(t *testing.T) {
 					assertNoneApplied(t, parts[0])
 					assertAllApplied(t, parts[1])
 				})
-
-				t.Run("case=applies namespace migrations on flag", func(t *testing.T) {
-					out := cmd.ExecNoErr(t, "up", "--"+FlagYes, "--"+FlagAllNamespace)
-
-					t.Cleanup(func() {
-						// migrate all down
-						t.Log(cmd.ExecNoErr(t, "down", "0", "--"+FlagYes))
-					})
-
-					parts := regexp.MustCompile("(?s)Current status:(.*)Successfully applied all migrations(.*)Going to migrate namespaces(.*)Successfully applied all migrations(.*)Current status(.*)Successfully applied all migrations(.*)").FindStringSubmatch(out)
-					require.Len(t, parts, 7)
-
-					assertNoneApplied(t, parts[1])
-					assertAllApplied(t, parts[2])
-					assertNoneApplied(t, parts[3])
-					assertAllApplied(t, parts[4])
-					assertNoneApplied(t, parts[5])
-					assertAllApplied(t, parts[6])
-				})
 			})
 		}
 	}

cmd/migrate/status.go

@@ -20,12 +20,12 @@ func newStatusCmd() *cobra.Command {
 		RunE: func(cmd *cobra.Command, _ []string) error {
 			ctx := cmd.Context()
 
-			reg, err := driver.NewDefaultRegistry(ctx, cmd.Flags())
+			reg, err := driver.NewDefaultRegistry(ctx, cmd.Flags(), true)
 			if err != nil {
 				return err
 			}
 
-			mb, err := reg.Migrator().MigrationBox(ctx)
+			mb, err := reg.MigrationBox()
 			if err != nil {
 				return err
 			}

cmd/migrate/up.go

@@ -9,8 +9,6 @@ import (
 
 	"github.com/spf13/pflag"
 
-	"github.com/pkg/errors"
-
 	"github.com/ory/x/cmdx"
 	"github.com/spf13/cobra"
 
@@ -34,12 +32,12 @@ func newUpCmd() *cobra.Command {
 		RunE: func(cmd *cobra.Command, _ []string) error {
 			ctx := cmd.Context()
 
-			reg, err := driver.NewDefaultRegistry(ctx, cmd.Flags())
+			reg, err := driver.NewDefaultRegistry(ctx, cmd.Flags(), true)
 			if err != nil {
 				return err
 			}
 
-			mb, err := reg.Migrator().MigrationBox(ctx)
+			mb, err := reg.MigrationBox()
 			if err != nil {
 				return err
 			}
@@ -48,35 +46,6 @@ func newUpCmd() *cobra.Command {
 				return err
 			}
 
-			if !allNamespaces {
-				// everything is done already
-				return nil
-			}
-
-			_, _ = fmt.Fprintln(cmd.OutOrStdout(), "\nGoing to migrate namespaces.")
-
-			nm, err := reg.Config().NamespaceManager()
-			if err != nil {
-				return errors.Wrap(err, "could not get the namespace manager")
-			}
-
-			nspaces, err := nm.Namespaces(cmd.Context())
-			if err != nil {
-				_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Could not get namespaces: %+v\n", err)
-				return cmdx.FailSilently(cmd)
-			}
-
-			for _, nspace := range nspaces {
-				mb, err := reg.NamespaceMigrator().NamespaceMigrationBox(ctx, nspace)
-				if err != nil {
-					return err
-				}
-
-				if err := BoxUp(cmd, mb, "[namespace="+nspace.Name+"] "); err != nil {
-					return err
-				}
-			}
-
 			return nil
 		},
 	}

cmd/namespace/migrate_down.go

@@ -1,56 +1,21 @@
 package namespace
 
 import (
-	"fmt"
-	"strconv"
-
 	"github.com/ory/keto/cmd/migrate"
 
-	"github.com/ory/x/cmdx"
 	"github.com/spf13/cobra"
-
-	"github.com/ory/keto/internal/driver"
 )
 
 func NewMigrateDownCmd() *cobra.Command {
 	cmd := &cobra.Command{
-		Use:   "down <namespace-name> <steps>",
-		Short: "Migrate a namespace down",
+		Deprecated: "This step is not necessary anymore, see https://github.com/ory/keto/pull/638",
+		Use:        "down <namespace-name> <steps>",
+		Short:      "Migrate a namespace down",
 		Long: "Migrate a namespace down.\n" +
 			"Pass 0 steps to fully migrate down.",
 		Args: cobra.ExactArgs(2),
-		RunE: func(cmd *cobra.Command, args []string) error {
-			ctx := cmd.Context()
-
-			steps, err := strconv.ParseInt(args[1], 0, 0)
-			if err != nil {
-				// return this error so it gets printed along the usage
-				return fmt.Errorf("malformed argument %s for <steps>: %+v", args[0], err)
-			}
-
-			reg, err := driver.NewDefaultRegistry(ctx, cmd.Flags())
-			if err != nil {
-				return err
-			}
-
-			nm, err := reg.Config().NamespaceManager()
-			if err != nil {
-				_, _ = fmt.Fprintf(cmd.OutOrStdout(), "Could not initialize the namespace manager: %+v\n", err)
-				return cmdx.FailSilently(cmd)
-			}
-
-			n, err := nm.GetNamespace(ctx, args[0])
-			if err != nil {
-				_, _ = fmt.Fprintf(cmd.OutOrStdout(), "Could not find the namespace with name \"%s\": %+v\n", args[0], err)
-				return cmdx.FailSilently(cmd)
-			}
-
-			mb, err := reg.NamespaceMigrator().NamespaceMigrationBox(ctx, n)
-			if err != nil {
-				return err
-			}
-
-			return migrate.BoxDown(cmd, mb, int(steps), "[namespace="+n.Name+"] ")
+		RunE: func(_ *cobra.Command, _ []string) error {
+			return nil
 		},
 	}
 

cmd/namespace/migrate_status.go

@@ -1,45 +1,19 @@
 package namespace
 
 import (
-	"fmt"
-
 	"github.com/ory/x/cmdx"
 	"github.com/spf13/cobra"
-
-	"github.com/ory/keto/cmd/migrate"
-	"github.com/ory/keto/internal/driver"
 )
 
 func NewMigrateStatusCmd() *cobra.Command {
 	cmd := &cobra.Command{
-		Use:   "status <namespace-name>",
-		Short: "Get the current namespace migration status",
-		Long:  "Get the current migration status of one specific namespace.\nDoes not apply any changes.",
-		Args:  cobra.ExactArgs(1),
-		RunE: func(cmd *cobra.Command, args []string) error {
-			reg, err := driver.NewDefaultRegistry(cmd.Context(), cmd.Flags())
-			if err != nil {
-				return err
-			}
-
-			nm, err := reg.Config().NamespaceManager()
-			if err != nil {
-				_, _ = fmt.Fprintf(cmd.OutOrStdout(), "Could not initialize the namespace manager: %+v\n", err)
-				return cmdx.FailSilently(cmd)
-			}
-
-			n, err := nm.GetNamespace(cmd.Context(), args[0])
-			if err != nil {
-				_, _ = fmt.Fprintf(cmd.OutOrStdout(), "Could not find the namespace with name \"%s\": %+v\n", args[0], err)
-				return cmdx.FailSilently(cmd)
-			}
-
-			mb, err := reg.NamespaceMigrator().NamespaceMigrationBox(cmd.Context(), n)
-			if err != nil {
-				return err
-			}
-
-			return migrate.BoxStatus(cmd, mb, "[namespace="+n.Name+"] ")
+		Deprecated: "This step is not necessary anymore, see https://github.com/ory/keto/pull/638",
+		Use:        "status <namespace-name>",
+		Short:      "Get the current namespace migration status",
+		Long:       "Get the current migration status of one specific namespace.\nDoes not apply any changes.",
+		Args:       cobra.ExactArgs(1),
+		RunE: func(_ *cobra.Command, _ []string) error {
+			return nil
 		},
 	}