chore: merge main to next (#5871)

### Definition of Ready

- [ ] I am happy with the code
- [ ] Short description of the feature/issue is added in the pr
description
- [ ] PR is linked to the corresponding user story
- [ ] Acceptance criteria are met
- [ ] All open todos and follow ups are defined in a new ticket and
justified
- [ ] Deviations from the acceptance criteria and design are agreed with
the PO and documented.
- [ ] No debug or dead code
- [ ] Critical parts are tested automatically
- [ ] Where possible E2E tests are implemented
- [ ] Documentation/examples are up-to-date
- [ ] All non-functional requirements are met
- [ ] Functionality of the acceptance criteria is checked manually on
the dev system.

.github/pull_request_template.md

@@ -1,5 +1,3 @@
-```[tasklist]
-
 ### Definition of Ready
 
 - [ ] I am happy with the code
@@ -14,4 +12,3 @@
 - [ ] Documentation/examples are up-to-date
 - [ ] All non-functional requirements are met
 - [ ] Functionality of the acceptance criteria is checked manually on the dev system.
-```

.github/workflows/integration.yml

@@ -0,0 +1,51 @@
+name: Integration tests
+
+on:
+  push:
+    tags-ignore:
+      - '**'
+  pull_request:
+    branches:
+      - '**'
+
+jobs:
+  run:
+    strategy:
+      matrix:
+        db: [cockroach, postgres]
+    runs-on: ubuntu-20.04
+    env:
+      DOCKER_BUILDKIT: 1
+      INTEGRATION_DB_FLAVOR: ${{ matrix.db }}
+      ZITADEL_MASTERKEY: MasterkeyNeedsToHave32Characters
+    steps:
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.19
+      - name: Source checkout
+        uses: actions/checkout@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          driver: docker
+          install: true
+      - name: Generate gRPC definitions
+        run: docker build -f build/grpc/Dockerfile -t zitadel-base:local .
+      - name: Copy gRPC definitions
+        run: docker build -f build/zitadel/Dockerfile . -t zitadel-go-base --target go-copy -o .
+      - name: Download Go modules
+        run: go mod download
+      - name: Start ${{ matrix.db }} database
+        run: docker compose -f internal/integration/config/docker-compose.yaml up --wait ${INTEGRATION_DB_FLAVOR}
+      - name: Run zitadel init and setup
+        run: |
+          go run main.go init --config internal/integration/config/zitadel.yaml --config internal/integration/config/${INTEGRATION_DB_FLAVOR}.yaml
+          go run main.go setup --masterkeyFromEnv --config internal/integration/config/zitadel.yaml --config internal/integration/config/${INTEGRATION_DB_FLAVOR}.yaml
+      - name: Run integration tests
+        run: go test -tags=integration -race -parallel 1 -v -coverprofile=profile.cov -coverpkg=./... ./internal/integration ./internal/api/grpc/...
+      - name: Publish go coverage
+        uses: codecov/codecov-action@v3.1.0
+        with:
+          file: profile.cov
+          name: integration-tests

.github/workflows/test-code.yml

@@ -44,7 +44,7 @@ jobs:
         uses: codecov/codecov-action@v3.1.0
         with:
           file: .artifacts/codecov/profile.cov
-          name: go-codecov
+          name: unit-tests
       # As goreleaser doesn't build a dockerfile in snapshot mode, we have to build it here
       - name: Build Docker Image
         run: docker build -t zitadel:pr --file build/Dockerfile .artifacts/zitadel

CONTRIBUTING.md

@@ -199,6 +199,21 @@ When you are happy with your changes, you can cleanup your environment.
 docker compose --file ./e2e/config/host.docker.internal/docker-compose.yaml down
 ```
 
+#### Integration tests
+
+In order to run the integrations tests for the gRPC API, PostgreSQL and CockroachDB must be started and initialized:
+
+```bash
+export INTEGRATION_DB_FLAVOR="cockroach" ZITADEL_MASTERKEY="MasterkeyNeedsToHave32Characters"
+docker compose -f internal/integration/config/docker-compose.yaml up --wait ${INTEGRATION_DB_FLAVOR}
+go run main.go init --config internal/integration/config/zitadel.yaml --config internal/integration/config/${INTEGRATION_DB_FLAVOR}.yaml
+go run main.go setup --masterkeyFromEnv --config internal/integration/config/zitadel.yaml --config internal/integration/config/${INTEGRATION_DB_FLAVOR}.yaml
+go test -tags=integration -race -parallel 1 ./internal/integration ./internal/api/grpc/...
+docker compose -f internal/integration/config/docker-compose.yaml down
+```
+
+The above can be repeated with `INTEGRATION_DB_FLAVOR="postgres"`.
+
 ### Console
 
 By executing the commands from this section, you run everything you need to develop the console locally.
@@ -315,13 +330,15 @@ docker compose down
 
 Project documentation is made with docusaurus and is located under [./docs](./docs).
 
-### Local Testing
+### Local Testing
+
 Please refer to the [README](./docs/README.md) for more information and local testing.
 
-### Style Guide
+### Style Guide
 
 - **Code with variables**: Make sure that code snippets can be used by setting environment variables, instead of manually replacing a placeholder.
 - **Embedded files**: When embedding mdx files, make sure the template ist prefixed by "_" (lowdash). The content will be rendered inside the parent page, but is not accessible individually (eg, by search).
+- **Embedded code**: You can embed code snippets from a repository. See the [plugin](https://github.com/saucelabs/docusaurus-theme-github-codeblock#usage) for usage.
 
 ### Docs Pull Request
 When making a pull request use `docs(<scope>): <short summary>` as title for the semantic release.

build/zitadel/generate-grpc.sh

@@ -18,8 +18,9 @@ protoc \
   --validate_out=lang=go:${GOPATH}/src \
   $(find ${PROTO_PATH} -iname *.proto)
 
-# install authoption proto compiler
+# install authoption and zitadel proto compiler
 go install ${ZITADEL_PATH}/internal/protoc/protoc-gen-auth
+go install ${ZITADEL_PATH}/internal/protoc/protoc-gen-zitadel
 
 # output folder for openapi v2
 mkdir -p ${OPENAPI_PATH}
@@ -79,7 +80,7 @@ protoc \
   --openapiv2_out ${OPENAPI_PATH} \
   --openapiv2_opt logtostderr=true \
   --openapiv2_opt allow_delete_body=true \
-  --auth_out=${GOPATH}/src \
+  --zitadel_out=${GOPATH}/src \
   --validate_out=lang=go:${GOPATH}/src \
   ${PROTO_PATH}/user/v2alpha/user_service.proto
 
@@ -91,8 +92,20 @@ protoc \
   --openapiv2_out ${OPENAPI_PATH} \
   --openapiv2_opt logtostderr=true \
   --openapiv2_opt allow_delete_body=true \
-  --auth_out=${GOPATH}/src \
+  --zitadel_out=${GOPATH}/src \
   --validate_out=lang=go:${GOPATH}/src \
   ${PROTO_PATH}/session/v2alpha/session_service.proto
 
+protoc \
+  -I=/proto/include \
+  --grpc-gateway_out ${GOPATH}/src \
+  --grpc-gateway_opt logtostderr=true \
+  --grpc-gateway_opt allow_delete_body=true \
+  --openapiv2_out ${OPENAPI_PATH} \
+  --openapiv2_opt logtostderr=true \
+  --openapiv2_opt allow_delete_body=true \
+  --zitadel_out=${GOPATH}/src \
+  --validate_out=lang=go:${GOPATH}/src \
+  ${PROTO_PATH}/settings/v2alpha/settings_service.proto
+
 echo "done generating grpc"

cmd/admin/admin.go

@@ -26,8 +26,8 @@ func New() *cobra.Command {
 	adminCMD.AddCommand(
 		initialise.New(),
 		setup.New(),
-		start.New(),
-		start.NewStartFromInit(),
+		start.New(nil),
+		start.NewStartFromInit(nil),
 		key.New(),
 	)
 

cmd/defaults.yaml

@@ -267,6 +267,7 @@ Console:
   LongCache:
     MaxAge: 12h
     SharedMaxAge: 168h #7d
+  InstanceManagementURL: ""
 
 Notification:
   Repository:

cmd/setup/03.go

@@ -77,6 +77,7 @@ func (mig *FirstInstance) Execute(ctx context.Context) error {
 		nil,
 		nil,
 		nil,
+		nil,
 	)
 
 	if err != nil {

cmd/setup/config_change.go

@@ -52,6 +52,7 @@ func (mig *externalConfigChange) Execute(ctx context.Context) error {
 		nil,
 		nil,
 		nil,
+		nil,
 	)
 
 	if err != nil {