feat: remove duplicate queries from whoami calls (#2995)

Introduces an expand API to the identity persister which greatly improves whoami performance.

Closes #2402

cmd/identities/delete_test.go

@@ -37,7 +37,7 @@ func TestDeleteCmd(t *testing.T) {
 		assert.Equal(t, i.ID.String(), gjson.Parse(stdOut).String())
 
 		// expect identity to be deleted
-		_, err := reg.Persister().GetIdentity(context.Background(), i.ID)
+		_, err := reg.Persister().GetIdentity(context.Background(), i.ID, identity.ExpandNothing)
 		assert.True(t, errors.Is(err, sqlcon.ErrNoRows))
 	})
 
@@ -49,7 +49,7 @@ func TestDeleteCmd(t *testing.T) {
 		assert.Equal(t, `["`+strings.Join(ids, "\",\"")+"\"]\n", stdOut)
 
 		for _, i := range is {
-			_, err := reg.Persister().GetIdentity(context.Background(), i.ID)
+			_, err := reg.Persister().GetIdentity(context.Background(), i.ID, identity.ExpandNothing)
 			assert.Error(t, err)
 		}
 	})

cmd/identities/import_test.go

@@ -10,6 +10,8 @@ import (
 	"os"
 	"testing"
 
+	"github.com/ory/kratos/identity"
+
 	"github.com/ory/kratos/cmd/identities"
 
 	"github.com/gofrs/uuid"
@@ -42,7 +44,7 @@ func TestImportCmd(t *testing.T) {
 
 		id, err := uuid.FromString(gjson.Get(stdOut, "id").String())
 		require.NoError(t, err)
-		_, err = reg.Persister().GetIdentity(context.Background(), id)
+		_, err = reg.Persister().GetIdentity(context.Background(), id, identity.ExpandNothing)
 		assert.NoError(t, err)
 	})
 
@@ -69,12 +71,12 @@ func TestImportCmd(t *testing.T) {
 
 		id, err := uuid.FromString(gjson.Get(stdOut, "0.id").String())
 		require.NoError(t, err)
-		_, err = reg.Persister().GetIdentity(context.Background(), id)
+		_, err = reg.Persister().GetIdentity(context.Background(), id, identity.ExpandNothing)
 		assert.NoError(t, err)
 
 		id, err = uuid.FromString(gjson.Get(stdOut, "1.id").String())
 		require.NoError(t, err)
-		_, err = reg.Persister().GetIdentity(context.Background(), id)
+		_, err = reg.Persister().GetIdentity(context.Background(), id, identity.ExpandNothing)
 		assert.NoError(t, err)
 	})
 
@@ -97,12 +99,12 @@ func TestImportCmd(t *testing.T) {
 
 		id, err := uuid.FromString(gjson.Get(stdOut, "0.id").String())
 		require.NoError(t, err)
-		_, err = reg.Persister().GetIdentity(context.Background(), id)
+		_, err = reg.Persister().GetIdentity(context.Background(), id, identity.ExpandNothing)
 		assert.NoError(t, err)
 
 		id, err = uuid.FromString(gjson.Get(stdOut, "1.id").String())
 		require.NoError(t, err)
-		_, err = reg.Persister().GetIdentity(context.Background(), id)
+		_, err = reg.Persister().GetIdentity(context.Background(), id, identity.ExpandNothing)
 		assert.NoError(t, err)
 	})
 
@@ -119,7 +121,7 @@ func TestImportCmd(t *testing.T) {
 
 		id, err := uuid.FromString(gjson.Get(stdOut, "id").String())
 		require.NoError(t, err)
-		_, err = reg.Persister().GetIdentity(context.Background(), id)
+		_, err = reg.Persister().GetIdentity(context.Background(), id, identity.ExpandNothing)
 		assert.NoError(t, err)
 	})
 }

driver/registry_default.go

@@ -723,6 +723,7 @@ func (m *RegistryDefault) VerificationTokenPersister() link.VerificationTokenPer
 func (m *RegistryDefault) VerificationCodePersister() code.VerificationCodePersister {
 	return m.Persister()
 }
+
 func (m *RegistryDefault) Persister() persistence.Persister {
 	return m.persister
 }

go.mod

@@ -4,6 +4,7 @@ go 1.19
 
 replace (
 	github.com/bradleyjkemp/cupaloy/v2 => github.com/aeneasr/cupaloy/v2 v2.6.1-0.20210924214125-3dfdd01210a3
+
 	github.com/gorilla/sessions => github.com/ory/sessions v1.2.2-0.20220110165800-b09c17334dc2
 	github.com/knadh/koanf => github.com/aeneasr/koanf v0.14.1-0.20211230115640-aa3902b3267a
 
@@ -41,8 +42,8 @@ require (
 	github.com/go-swagger/go-swagger v0.30.3
 	github.com/gobuffalo/fizz v1.14.4
 	github.com/gobuffalo/httptest v1.5.2
-	github.com/gobuffalo/pop/v6 v6.0.8
-	github.com/gofrs/uuid v4.3.0+incompatible
+	github.com/gobuffalo/pop/v6 v6.1.1-0.20230102153939-35967190380a
+	github.com/gofrs/uuid v4.3.1+incompatible
 	github.com/golang-jwt/jwt/v4 v4.1.0
 	github.com/golang/gddo v0.0.0-20190904175337-72a348e765d2
 	github.com/golang/mock v1.6.0
@@ -77,11 +78,12 @@ require (
 	github.com/ory/jsonschema/v3 v3.0.7
 	github.com/ory/mail/v3 v3.0.0
 	github.com/ory/nosurf v1.2.7
-	github.com/ory/x v0.0.523
+	github.com/ory/x v0.0.527
 	github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2
 	github.com/pkg/errors v0.9.1
 	github.com/pquerna/otp v1.4.0
 	github.com/rs/cors v1.8.2
+	github.com/samber/lo v1.37.0
 	github.com/sirupsen/logrus v1.9.0
 	github.com/slack-go/slack v0.7.4
 	github.com/spf13/cobra v1.6.1
@@ -313,6 +315,7 @@ require (
 	go.uber.org/atomic v1.10.0 // indirect
 	go.uber.org/multierr v1.7.0 // indirect
 	go.uber.org/zap v1.17.0 // indirect
+	golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 // indirect
 	golang.org/x/mod v0.7.0 // indirect
 	golang.org/x/sys v0.3.0 // indirect
 	golang.org/x/term v0.3.0 // indirect

go.sum

@@ -517,8 +517,8 @@ github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGt
 github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0=
 github.com/gobuffalo/plush/v4 v4.1.16 h1:Y6jVVTLdg1BxRXDIbTJz+J8QRzEAtv5ZwYpGdIFR7VU=
 github.com/gobuffalo/plush/v4 v4.1.16/go.mod h1:6t7swVsarJ8qSLw1qyAH/KbrcSTwdun2ASEQkOznakg=
-github.com/gobuffalo/pop/v6 v6.0.8 h1:9+5ShHYh3x9NDFCITfm/gtKDDRSgOwiY7kA0Hf7N9aQ=
-github.com/gobuffalo/pop/v6 v6.0.8/go.mod h1:f4JQ4Zvkffcevz+t+XAwBLStD7IQs19DiIGIDFYw1eA=
+github.com/gobuffalo/pop/v6 v6.1.1-0.20230102153939-35967190380a h1:O6PLVjaR9oDlaU+WOwfwNLvMFik0zdmvAyExIbJhGcI=
+github.com/gobuffalo/pop/v6 v6.1.1-0.20230102153939-35967190380a/go.mod h1:Y3nCI31Zx40ffCnpQsYCMOWvR6f17K+IukEg0EHNxaQ=
 github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw=
 github.com/gobuffalo/tags/v3 v3.1.4 h1:X/ydLLPhgXV4h04Hp2xlbI2oc5MDaa7eub6zw8oHjsM=
 github.com/gobuffalo/tags/v3 v3.1.4/go.mod h1:ArRNo3ErlHO8BtdA0REaZxijuWnWzF6PUXngmMXd2I0=
@@ -533,8 +533,8 @@ github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw=
 github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
 github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gofrs/uuid v4.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
-github.com/gofrs/uuid v4.3.0+incompatible h1:CaSVZxm5B+7o45rtab4jC2G37WGYX1zQfuU2i6DSvnc=
-github.com/gofrs/uuid v4.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
+github.com/gofrs/uuid v4.3.1+incompatible h1:0/KbAdpx3UXAx1kEOWHJeOkpbgRFGHVgv+CFIY7dBJI=
+github.com/gofrs/uuid v4.3.1+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
@@ -806,7 +806,6 @@ github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLf
 github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
 github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf h1:FtEj8sfIcaaBfAKrE1Cwb61YDtYq9JxChK1c7AKce7s=
 github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf/go.mod h1:yrqSXGoD/4EKfF26AOGzscPOgTTJcyAwM2rpixWT+t4=
-github.com/instana/testify v1.6.2-0.20200721153833-94b1851f4d65 h1:T25FL3WEzgmKB0m6XCJNZ65nw09/QIp3T1yXr487D+A=
 github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo=
 github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
 github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8=
@@ -1141,8 +1140,8 @@ github.com/ory/sessions v1.2.2-0.20220110165800-b09c17334dc2 h1:zm6sDvHy/U9XrGpi
 github.com/ory/sessions v1.2.2-0.20220110165800-b09c17334dc2/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
 github.com/ory/viper v1.7.5 h1:+xVdq7SU3e1vNaCsk/ixsfxE4zylk1TJUiJrY647jUE=
 github.com/ory/viper v1.7.5/go.mod h1:ypOuyJmEUb3oENywQZRgeAMwqgOyDqwboO1tj3DjTaM=
-github.com/ory/x v0.0.523 h1:vn8e+8tV3RqD8RlvoE6lLPUnjpjua1ExJDMFy3Z5TAQ=
-github.com/ory/x v0.0.523/go.mod h1:ayJio5x/fK4RwTgfgzs3JetOaaOSxso9hQjc3mFY8z0=
+github.com/ory/x v0.0.527 h1:K6MmsYqT1NMb8VQ4hhn9q6NnrnecwNQJXc1bEoixQ8Y=
+github.com/ory/x v0.0.527/go.mod h1:XBqhPZRppPHTxtsE0l0oI/B2Onf1QJtMRGPh3CpEpA0=
 github.com/otiai10/copy v1.2.0/go.mod h1:rrF5dJ5F0t/EWSYODDu4j9/vEeYHMkc8jt0zJChqQWw=
 github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE=
 github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs=
@@ -1269,6 +1268,8 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
+github.com/samber/lo v1.37.0 h1:XjVcB8g6tgUp8rsPsJ2CvhClfImrpL04YpQHXeHPhRw=
+github.com/samber/lo v1.37.0/go.mod h1:9vaz2O4o8oOnK23pd2TrXufcbdbJIa3b6cstBWKpopA=
 github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
 github.com/sassoftware/go-rpmutils v0.0.0-20190420191620-a8f1baeba37b/go.mod h1:am+Fp8Bt506lA3Rk3QCmSqmYmLMnPDhdDUcosQCAx+I=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
@@ -1338,7 +1339,6 @@ github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tL
 github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
 github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
 github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
-github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM=
 github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA=
 github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
@@ -1600,6 +1600,8 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
 golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw=
+golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 h1:3MTrJm4PyNL9NBqvYDSj3DHl46qQakyfqfWo4jgfaEM=
+golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=

identity/.snapshots/TestHandler-case=should_be_able_to_import_users-with_cleartext_password_and_oidc_credentials.json

@@ -2,10 +2,6 @@
   "credentials": {
     "oidc": {
       "type": "oidc",
-      "identifiers": [
-        "google:import-2",
-        "github:import-2"
-      ],
       "config": {
         "providers": [
           {

credentialmigrate/.snapshots/TestUpgradeCredentials-type=webauthn-from=v0.json → identity/.snapshots/TestUpgradeCredentials-type=webauthn-from=v0.json

@@ -20,6 +20,19 @@
             "display_name": "asdf",
             "added_at": "2022-02-28T16:40:39Z",
             "is_passwordless": false
+          },
+          {
+            "id": "1Q4LaIJ9NiqS1r0CQpWY+K0gMvhOq4yk5BHuO/YlitcurSpBK7weDXOvBcuN4lvn6DAmjGfmj/J/6bpOmtdT8Q==",
+            "public_key": "pQECAyYgASFYILAYFLoH1T8bQMSbPrNBCMMS5U7OFWRwv2U+GkAoiBADIlggBv+8ni7XVZYBB8ufMbP/d9fDxbmOkVVHOgcJifnoOR4=",
+            "attestation_type": "none",
+            "authenticator": {
+              "aaguid": "AAAAAAAAAAAAAAAAAAAAAA==",
+              "sign_count": 4,
+              "clone_warning": false
+            },
+            "display_name": "asdf",
+            "added_at": "2022-02-28T16:40:39Z",
+            "is_passwordless": false
           }
         ],
         "user_handle": "TWT6CCD8RQ2+vevXx7biSQ=="

identity/credentials.go

@@ -8,6 +8,8 @@ import (
 	"reflect"
 	"time"
 
+	"github.com/gobuffalo/pop/v6"
+
 	"github.com/ory/kratos/ui/node"
 
 	"github.com/gofrs/uuid"
@@ -84,8 +86,6 @@ const (
 type Credentials struct {
 	ID uuid.UUID `json:"-" db:"id"`
 
-	CredentialTypeID uuid.UUID `json:"-" db:"identity_credential_type_id"`
-
 	// Type discriminates between different types of credentials.
 	Type CredentialsType `json:"type" db:"-"`
 
@@ -107,6 +107,30 @@ type Credentials struct {
 	// UpdatedAt is a helper struct field for gobuffalo.pop.
 	UpdatedAt time.Time `json:"updated_at" db:"updated_at"`
 	NID       uuid.UUID `json:"-"  faker:"-" db:"nid"`
+
+	IdentityCredentialTypeID uuid.UUID                      `json:"-" db:"identity_credential_type_id"`
+	IdentityCredentialType   CredentialsTypeTable           `json:"-" faker:"-" belongs_to:"identity_credential_types"`
+	CredentialIdentifiers    CredentialIdentifierCollection `json:"-" faker:"-" has_many:"identity_credential_identifiers" fk_id:"identity_credential_id" order_by:"id asc"`
+}
+
+func (c *Credentials) AfterEagerFind(tx *pop.Connection) error {
+	return c.setCredentials()
+}
+
+func (c *Credentials) setCredentials() error {
+	c.Type = c.IdentityCredentialType.Name
+	c.Identifiers = make([]string, 0, len(c.CredentialIdentifiers))
+	for _, id := range c.CredentialIdentifiers {
+		if c.NID != id.NID {
+			continue
+		}
+		c.Identifiers = append(c.Identifiers, id.Identifier)
+	}
+	return nil
+}
+
+func (c Credentials) TableName(ctx context.Context) string {
+	return "identity_credentials"
 }
 
 type (
@@ -158,10 +182,6 @@ func (c CredentialsCollection) TableName(ctx context.Context) string {
 	return "identity_credentials"
 }
 
-func (c Credentials) TableName(ctx context.Context) string {
-	return "identity_credentials"
-}
-
 func (c CredentialIdentifierCollection) TableName(ctx context.Context) string {
 	return "identity_credential_identifiers"
 }