Merge branch 'main' into rc

.dockerignore

@@ -7,6 +7,7 @@
 /k8s/
 /node_modules/
 /console/src/app/proto/generated/
+/console/.angular
 /console/tmp/
 .releaserc.js
 changelog.config.js
@@ -18,3 +19,4 @@ pkg/grpc/*/*.pb.*
 pkg/grpc/*/*.swagger.json
 .goreleaser.yaml
 .artifacts/
+.vscode

.github/pull_request_template.md

@@ -7,6 +7,7 @@
 - [ ] All open todos and follow ups are defined in a new ticket and justified
 - [ ] Deviations from the acceptance criteria and design are agreed with the PO and documented.
 - [ ] No debug or dead code
+- [ ] My code has no repetitions
 - [ ] Critical parts are tested automatically
 - [ ] Where possible E2E tests are implemented
 - [ ] Documentation/examples are up-to-date

.github/workflows/integration.yml

@@ -43,7 +43,7 @@ jobs:
           go run main.go init --config internal/integration/config/zitadel.yaml --config internal/integration/config/${INTEGRATION_DB_FLAVOR}.yaml
           go run main.go setup --masterkeyFromEnv --config internal/integration/config/zitadel.yaml --config internal/integration/config/${INTEGRATION_DB_FLAVOR}.yaml
       - name: Run integration tests
-        run: go test -tags=integration -race -parallel 1 -v -coverprofile=profile.cov -coverpkg=./... ./internal/integration ./internal/api/grpc/...
+        run: go test -tags=integration -race -parallel 1 -v -coverprofile=profile.cov -coverpkg=./internal/...,./cmd/... ./internal/integration ./internal/api/grpc/...
       - name: Publish go coverage
         uses: codecov/codecov-action@v3.1.0
         with:

.github/workflows/zitadel.yml

@@ -80,7 +80,7 @@ jobs:
           name: go-codecov
       - name: Bump Chart Version
         uses: peter-evans/repository-dispatch@v2
-        if: steps.semantic.outputs.new_release_published == 'true' && github.ref == 'refs/heads/main'
+        if: steps.semantic.outputs.new_release_published == 'true' && github.ref == 'refs/heads/next'
         with:
           token: ${{ steps.generate-token.outputs.token }}
           repository: zitadel/zitadel-charts

.goreleaser.yaml

@@ -18,6 +18,7 @@ before:
     - docker build -f build/grpc/Dockerfile -t zitadel-base:local .
     - docker build -f build/zitadel/Dockerfile . -t zitadel-go-test --target go-codecov -o .artifacts/codecov
     - docker build -f build/zitadel/Dockerfile . -t zitadel-go-base --target go-copy -o .artifacts/grpc/go-client
+    - sh -c "find pkg/grpc -name '*.pb*.go' -delete"
     - sh -c "cp -r .artifacts/grpc/go-client/* ."
     - docker build -f build/console/Dockerfile . -t zitadel-npm-console --target angular-export -o .artifacts/console
     - sh -c "cp -r .artifacts/console/* internal/api/ui/console/static/"

.releaserc.js

@@ -1,7 +1,7 @@
 module.exports = {
     branches: [
-        {name: 'main'},
-        {name: 'next'},
+        { name: 'main' },
+        { name: 'next' },
     ],
     plugins: [
         "@semantic-release/commit-analyzer"

CONTRIBUTING.md

@@ -36,7 +36,7 @@ We strongly recommend to [talk to us](https://zitadel.com/contact) before you st
 
 We accept contributions through pull requests. You need a github account for that. If you are unfamiliar with git have a look at Github's documentation on [creating forks](https://help.github.com/articles/fork-a-repo) and [creating pull requests](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request-from-a-fork). Please draft the pull request as soon as possible. Go through the following checklist before you submit the final pull request:
 
-### Submit a Pull Request (PR)
+### Submit a pull request (PR)
 
 1. [Fork](https://docs.github.com/en/get-started/quickstart/fork-a-repo) the [zitadel/zitadel](https://github.com/zitadel/zitadel) repository on GitHub
 2. On your fork, commit your changes to a new branch
@@ -59,14 +59,14 @@ We accept contributions through pull requests. You need a github account for tha
 
 8. On GitHub, [send a pull request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/requesting-a-pull-request-review) to `zitadel:main`. Request review from one of the maintainers.
 
-### Reviewing a Pull Request
+### Review a pull request
 
 The reviewers will provide you feedback and approve your changes as soon as they are satisfied. If we ask you for changes in the code, you can follow the [GitHub Guide](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/reviewing-changes-in-pull-requests/incorporating-feedback-in-your-pull-request) to incorporate feedback in your pull request.
 
 <!-- TODO: how to do this via git -->
 <!-- TODO: change commit message via git -->
 
-### Commit Messages
+### Commit messages
 
 Make sure you use [semantic release messages format](https://github.com/angular/angular.js/blob/master/DEVELOPERS.md#type).
 
@@ -84,7 +84,7 @@ Must be one of the following:
 
 This is optional to indicate which component is affected. In doubt, leave blank (`<type>: <short summary>`)
 
-#### Short Summary
+#### Short summary
 
 Provide a brief description of the change.
 
@@ -107,7 +107,7 @@ We add the label "good first issue" for problems we think are a good starting po
 - [Issues for first time contributors](https://github.com/zitadel/zitadel/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22)
 - [All issues](https://github.com/zitadel/zitadel/issues)
 
-### Backend / Login
+### Backend/login
 
 By executing the commands from this section, you run everything you need to develop the ZITADEL backend locally.
 Using [Docker Compose](https://docs.docker.com/compose/), you run a [CockroachDB](https://www.cockroachlabs.com/docs/stable/start-a-local-cluster-in-docker-mac.html) on your local machine.
@@ -231,7 +231,6 @@ The commands in this section are tested against the following software versions:
 - [Node version v16.17.0](https://nodejs.org/en/download/)
 - [npm version 8.18.0](https://docs.npmjs.com/try-the-latest-stable-version-of-npm)
 - [Cypress runtime dependencies](https://docs.cypress.io/guides/continuous-integration/introduction#Dependencies)
-- [curl version 7.58.0](https://curl.se/download.html)
 
 <details>
   <summary>Note for WSL2 on Windows 10</summary>
@@ -269,18 +268,17 @@ To allow console access via http://localhost:4200, you have to configure the ZIT
 You can run the local console development server now.
 
 ```bash
-# Console loads its target environment from the file console/src/assets/environment.json.
-# Load it from the backend.
-curl http://localhost:8080/ui/console/assets/environment.json > ./src/assets/environment.json
+# Install npm dependencies
+npm install
 
 # Generate source files from Protos
 npm run generate
 
-# Install npm dependencies
-npm install
-
 # Start the server
 npm start
+
+# If you don't want to develop against http://localhost:8080, you can use another environment
+ENVIRONMENT_JSON_URL=https://my-cloud-instance-abcdef.zitadel.cloud/ui/console/assets/environment.json npm start
 ```
 
 Navigate to http://localhost:4200/.
@@ -326,25 +324,36 @@ When you are happy with your changes, you can format your code and cleanup your
 docker compose down
 ```
 
-## Contribute Docs
+## Contribute docs
 
 Project documentation is made with docusaurus and is located under [./docs](./docs).
 
-### Local Testing
+### Local testing
 
 Please refer to the [README](./docs/README.md) for more information and local testing.
 
-### Style Guide
+### Style guide
 
 - **Code with variables**: Make sure that code snippets can be used by setting environment variables, instead of manually replacing a placeholder.
 - **Embedded files**: When embedding mdx files, make sure the template ist prefixed by "_" (lowdash). The content will be rendered inside the parent page, but is not accessible individually (eg, by search).
+- **Don't repeat yourself**: When using the same content in multiple places, save and manage the content as separate file and make use of embedded files to import it into other docs pages.
 - **Embedded code**: You can embed code snippets from a repository. See the [plugin](https://github.com/saucelabs/docusaurus-theme-github-codeblock#usage) for usage.
 
-### Docs Pull Request
+Following the [Google style guide](https://developers.google.com/style) is highly recommended. Its clear and concise guidelines ensure consistency and effective communication within the wider developer community.
+
+The style guide covers a lot of material, so their [highlights](https://developers.google.com/style/highlights) page provides an overview of its most important points. Some of the points stated in the highlights that we care about most are given below: 
+
+- Be conversational and friendly without being frivolous.
+- Use sentence case for document titles and section headings.
+- Use active voice: make clear who's performing the action.
+- Use descriptive link text.
+
+### Docs pull request
+
 When making a pull request use `docs(<scope>): <short summary>` as title for the semantic release.
 Scope can be left empty (omit the brackets) or refer to the top navigation sections.
 
-## Contribute Internationalization
+## Contribute internationalization
 
 ZITADEL loads translations from four files:
 
@@ -364,7 +373,7 @@ You can find an installation guide for all the different environments here:
 
 - Please read [Security Policy](./SECURITY.md).
 
-## Product Management
+## Product management
 
 The ZITADEL Team works with an agile product management methodology.
 You can find all the issues prioritized and ordered in the [product board](https://github.com/orgs/zitadel/projects/2/views/1).
@@ -388,10 +397,10 @@ The state should reflect the progress of the issue and what is going on right no
 
 - **No status**: Issue just got added and has to be looked at.
 - **🧐 Investigating**: We are currently investigating to find out what the problem is, which priority it should have and what has to be implemented. Or we need some more information from the author.
-- **📨 Product Backlog**: If an issue is in the backlog, it is not currently being worked on. These are recorded so that they can be worked on in the future. Issues with this state do not have to be completely defined yet.
-- **📝 Prioritized Product Backlog**: An issue with the state "Prioritized Backlog" is ready for the refinement from the perspective of the product owner (PO) to implement. This means the developer can find all the relevant information and acceptance criteria in the issue.
+- **📨 Product backlog**: If an issue is in the backlog, it is not currently being worked on. These are recorded so that they can be worked on in the future. Issues with this state do not have to be completely defined yet.
+- **📝 Prioritized product backlog**: An issue with the state "Prioritized Backlog" is ready for the refinement from the perspective of the product owner (PO) to implement. This means the developer can find all the relevant information and acceptance criteria in the issue.
 - **🔖 Ready**: The issue is ready to take into a sprint. Difference to "prioritized..." is that the complexity is defined by the team.
-- **📋 Sprint Backlog**: The issue is scheduled for the current sprint.
+- **📋 Sprint backlog**: The issue is scheduled for the current sprint.
 - **🏗 In progress**: Someone is working on this issue right now. The issue will get an assignee as soon as it is in progress.
 - **👀 In review**: The issue is in review. Please add someone to review your issue or let us know that it is ready to review with a comment on your pull request.
 - **✅ Done**: The issue is implemented and merged to main.
@@ -413,7 +422,7 @@ Everything that is higher than 8 should be split in smaller parts.
 
 **1**, **2**, **3**, **5**, **8**, **13**
 
-### About the Labels
+### About the labels
 
 There are a few general labels that don't belong to a specific category.
 

README.md

@@ -94,7 +94,7 @@ Authentication
 - Single Sign On (SSO)
 - Passwordless with FIDO2 support (Including Passkeys)
 - Username / Password
-- Multifactor authentication with OTP, U2F, SMS
+- Multifactor authentication with OTP, U2F
 - LDAP
 - [OpenID Connect certified](https://openid.net/certification/#OPs) => [OIDC Endpoints](https://zitadel.com/docs/apis/openidoauth/endpoints)
 - [SAML 2.0](http://docs.oasis-open.org/security/saml/Post2.0/sstc-saml-tech-overview-2.0.html) => [SAML Endpoints](https://zitadel.com/docs/apis/saml/endpoints)

cmd/defaults.yaml

@@ -735,6 +735,7 @@ InternalAuthZ:
         - "user.grant.delete"
         - "user.membership.read"
         - "user.credential.write"
+        - "user.passkey.write"
         - "policy.read"
         - "policy.write"
         - "policy.delete"
@@ -811,6 +812,7 @@ InternalAuthZ:
         - "user.grant.delete"
         - "user.membership.read"
         - "user.credential.write"
+        - "user.passkey.write"
         - "policy.read"
         - "policy.write"
         - "policy.delete"
@@ -847,6 +849,7 @@ InternalAuthZ:
         - "user.grant.write"
         - "user.grant.delete"
         - "user.membership.read"
+        - "user.passkey.write"
         - "project.read"
         - "project.member.read"
         - "project.role.read"
@@ -882,6 +885,7 @@ InternalAuthZ:
         - "user.grant.delete"
         - "user.membership.read"
         - "user.credential.write"
+        - "user.passkey.write"
         - "policy.read"
         - "policy.write"
         - "policy.delete"

cmd/setup/10.go

@@ -13,11 +13,11 @@ import (
 )
 
 var (
-	//go:embed 10_create_temp_table.sql
+	//go:embed 10/10_create_temp_table.sql
 	correctCreationDate10CreateTable string
-	//go:embed 10_fill_table.sql
+	//go:embed 10/10_fill_table.sql
 	correctCreationDate10FillTable string
-	//go:embed 10_update.sql
+	//go:embed 10/10_update.sql
 	correctCreationDate10Update string
 )
 

cmd/setup/11.go

@@ -0,0 +1,32 @@
+package setup
+
+import (
+	"context"
+	_ "embed"
+
+	"github.com/zitadel/zitadel/internal/database"
+)
+
+var (
+	//go:embed 11.sql
+	addEventCreatedAt string
+)
+
+type AddEventCreatedAt struct {
+	step10   *CorrectCreationDate
+	dbClient *database.DB
+}
+
+func (mig *AddEventCreatedAt) Execute(ctx context.Context) error {
+	// execute step 10 again because events created after the first execution of step 10
+	// could still have the wrong ordering of sequences and creation date
+	if err := mig.step10.Execute(ctx); err != nil {
+		return err
+	}
+	_, err := mig.dbClient.ExecContext(ctx, addEventCreatedAt)
+	return err
+}
+
+func (mig *AddEventCreatedAt) String() string {
+	return "11_event_created_at"
+}

cmd/setup/11.sql

@@ -0,0 +1,15 @@
+BEGIN;
+-- create table with empty created_at
+ALTER TABLE eventstore.events ADD COLUMN created_at TIMESTAMPTZ DEFAULT NULL;
+COMMIT;
+
+BEGIN;
+-- backfill created_at
+UPDATE eventstore.events SET created_at = creation_date WHERE created_at IS NULL;
+COMMIT;
+
+BEGIN;
+-- set column rules
+ALTER TABLE eventstore.events ALTER COLUMN created_at SET DEFAULT clock_timestamp();
+ALTER TABLE eventstore.events ALTER COLUMN created_at SET NOT NULL;
+COMMIT;

cmd/setup/config.go

@@ -66,6 +66,7 @@ type Steps struct {
 	s8AuthTokens         *AuthTokenIndexes
 	s9EventstoreIndexes2 *EventstoreIndexesNew
 	CorrectCreationDate  *CorrectCreationDate
+	s11AddEventCreatedAt *AddEventCreatedAt
 }
 
 type encryptionKeyConfig struct {

cmd/setup/setup.go

@@ -91,6 +91,7 @@ func Setup(config *Config, steps *Steps, masterKey string) {
 	steps.s8AuthTokens = &AuthTokenIndexes{dbClient: dbClient}
 	steps.s9EventstoreIndexes2 = New09(dbClient)
 	steps.CorrectCreationDate.dbClient = dbClient
+	steps.s11AddEventCreatedAt = &AddEventCreatedAt{dbClient: dbClient, step10: steps.CorrectCreationDate}
 
 	err = projection.Create(ctx, dbClient, eventstoreClient, config.Projections, nil, nil)
 	logging.OnError(err).Fatal("unable to start projections")
@@ -128,6 +129,8 @@ func Setup(config *Config, steps *Steps, masterKey string) {
 	logging.OnError(err).Fatal("unable to migrate step 9")
 	err = migration.Migrate(ctx, eventstoreClient, steps.CorrectCreationDate)
 	logging.OnError(err).Fatal("unable to migrate step 10")
+	err = migration.Migrate(ctx, eventstoreClient, steps.s11AddEventCreatedAt)
+	logging.OnError(err).Fatal("unable to migrate step 11")
 
 	for _, repeatableStep := range repeatableSteps {
 		err = migration.Migrate(ctx, eventstoreClient, repeatableStep)

cmd/start/start.go

@@ -38,6 +38,7 @@ import (
 	"github.com/zitadel/zitadel/internal/api/grpc/user/v2"
 	http_util "github.com/zitadel/zitadel/internal/api/http"
 	"github.com/zitadel/zitadel/internal/api/http/middleware"
+	"github.com/zitadel/zitadel/internal/api/idp"
 	"github.com/zitadel/zitadel/internal/api/oidc"
 	"github.com/zitadel/zitadel/internal/api/robots_txt"
 	"github.com/zitadel/zitadel/internal/api/saml"
@@ -306,9 +307,8 @@ func startAPIs(
 		http_util.WithNonHttpOnly(),
 		http_util.WithMaxAge(int(math.Floor(config.Quotas.Access.ExhaustedCookieMaxAge.Seconds()))),
 	)
-	limitingAccessInterceptor := middleware.NewAccessInterceptor(accessSvc, exhaustedCookieHandler, config.Quotas.Access, false)
-	nonLimitingAccessInterceptor := middleware.NewAccessInterceptor(accessSvc, nil, config.Quotas.Access, true)
-	apis, err := api.New(ctx, config.Port, router, queries, verifier, config.InternalAuthZ, tlsConfig, config.HTTP2HostHeader, config.HTTP1HostHeader, accessSvc, exhaustedCookieHandler, config.Quotas.Access)
+	limitingAccessInterceptor := middleware.NewAccessInterceptor(accessSvc, exhaustedCookieHandler, config.Quotas.Access)
+	apis, err := api.New(ctx, config.Port, router, queries, verifier, config.InternalAuthZ, tlsConfig, config.HTTP2HostHeader, config.HTTP1HostHeader, limitingAccessInterceptor)
 	if err != nil {
 		return fmt.Errorf("error creating api %w", err)
 	}
@@ -332,7 +332,7 @@ func startAPIs(
 	if err := apis.RegisterServer(ctx, auth.CreateServer(commands, queries, authRepo, config.SystemDefaults, keys.User, config.ExternalSecure, config.AuditLogRetention)); err != nil {
 		return err
 	}
-	if err := apis.RegisterService(ctx, user.CreateServer(commands, queries, keys.User)); err != nil {
+	if err := apis.RegisterService(ctx, user.CreateServer(commands, queries, keys.User, keys.IDPConfig, idp.CallbackURL(config.ExternalSecure))); err != nil {
 		return err
 	}
 	if err := apis.RegisterService(ctx, session.CreateServer(commands, queries, permissionCheck)); err != nil {
@@ -345,6 +345,8 @@ func startAPIs(
 	assetsCache := middleware.AssetsCacheInterceptor(config.AssetStorage.Cache.MaxAge, config.AssetStorage.Cache.SharedMaxAge)
 	apis.RegisterHandlerOnPrefix(assets.HandlerPrefix, assets.NewHandler(commands, verifier, config.InternalAuthZ, id.SonyFlakeGenerator(), store, queries, middleware.CallDurationHandler, instanceInterceptor.Handler, assetsCache.Handler, limitingAccessInterceptor.Handle))
 
+	apis.RegisterHandlerOnPrefix(idp.HandlerPrefix, idp.NewHandler(commands, queries, keys.IDPConfig, config.ExternalSecure, instanceInterceptor.Handler))
+
 	userAgentInterceptor, err := middleware.NewUserAgentHandler(config.UserAgentCookie, keys.UserAgentCookieKey, id.SonyFlakeGenerator(), config.ExternalSecure, login.EndpointResources)
 	if err != nil {
 		return err
@@ -376,7 +378,7 @@ func startAPIs(
 	}
 	apis.RegisterHandlerOnPrefix(saml.HandlerPrefix, samlProvider.HttpHandler())
 
-	c, err := console.Start(config.Console, config.ExternalSecure, oidcProvider.IssuerFromRequest, middleware.CallDurationHandler, instanceInterceptor.Handler, nonLimitingAccessInterceptor.Handle, config.CustomerPortal)
+	c, err := console.Start(config.Console, config.ExternalSecure, oidcProvider.IssuerFromRequest, middleware.CallDurationHandler, instanceInterceptor.Handler, limitingAccessInterceptor, config.CustomerPortal)
 	if err != nil {
 		return fmt.Errorf("unable to start console: %w", err)
 	}

console/angular.json

@@ -35,7 +35,8 @@
               "codemirror/mode/javascript/javascript",
               "codemirror/mode/xml/xml",
               "file-saver",
-              "qrcode"
+              "qrcode",
+              "codemirror"
             ]
           },
           "configurations": {