With ng 16 (#5878)

* feat: enable grpc server reflection

* add server reflection to Probes list

* fix(eventstore): new column to test clock_timestamp()

* fix(eventstore): use creation_date for existing columns

* fix(eventstore): backfill column

* fix(step11): execute step 10 to make sure events are in correct order

* chore: speed up local console docker build (#5824)

Co-authored-by: Livio Spring <livio.a@gmail.com>

* docs(legal): editorial changes (#5828)

* docs(legal): editorial changes

* remove from tos

* docs: add generated docs to gitignore (#5827)

* fix: set exhausted cookie with env json (#5868)

* fix: set exhausted cookie with env json

* lint

* docs(nginx): fix nginx directory, system api proto description (#5809)

Co-authored-by: Elio Bischof <elio@zitadel.com>

* docs(contributing): don't repeat yourself (#5869)

* ng 16

* material cdk

* module imports, rmn @accounts

* deps

* lint

* buf

* cache to registry

* push

* var

* ref

* vars

* so?

* next

* new buildx

* params

---------

Co-authored-by: Tim Möhlmann <tim+github@zitadel.com>
Co-authored-by: Elio Bischof <elio@zitadel.com>
Co-authored-by: Livio Spring <livio.a@gmail.com>
Co-authored-by: mffap <mpa@zitadel.com>
Co-authored-by: kotoriのねこ <minamiktr@outlook.com>
Co-authored-by: Max Peintner <max@caos.ch>

.dockerignore

@@ -17,4 +17,5 @@ SECURITY.md
 **/.angular
 **/console/node_modules
 **/console/src/app/proto/generated/
-**/console/tmp/
+**/console/tmp/
+**/.vscode

.github/workflows/all.yml

@@ -9,6 +9,11 @@ permissions:
 
 env:
   go_version: '1.20'
+  cache_from_params: type=registry,ref=ghcr.io/zitadel/build-cache
+  cache_to_params: type=registry,ref=ghcr.io/zitadel/build-cache,mode=max
+  tag_name: ghcr.io/zitadel/test:${{ github.sha }}
+  buildx_driver_opts: |
+      image=moby/buildkit:v0.11.2
 
 jobs:
 
@@ -24,6 +29,15 @@ jobs:
       -
         name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
+        with:
+          driver-opts: ${{env.buildx_driver_opts}}
+      - 
+        name: Login to Docker registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
       -
         name: Build
         uses: docker/build-push-action@v4
@@ -32,10 +46,8 @@ jobs:
           file: ./build/workflow.Dockerfile
           target: core-build
           push: false
-          cache-to: |
-            type=gha
-          cache-from: |
-            type=gha
+          cache-to: ${{env.cache_to_params}}
+          cache-from: ${{env.cache_from_params}}
           github-token: ${{ github.token }}
 
   console:
@@ -50,6 +62,15 @@ jobs:
       -
         name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
+        with:
+          driver-opts: ${{env.buildx_driver_opts}}
+      - 
+        name: Login to Docker registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
       -
         name: Build
         uses: docker/build-push-action@v4
@@ -58,11 +79,11 @@ jobs:
           file: ./build/workflow.Dockerfile
           target: console
           push: false
-          cache-to: |
-            type=gha
-          cache-from: |
-            type=gha
+          cache-to:  ${{env.cache_to_params}}
+          cache-from: ${{env.cache_from_params}}
           github-token: ${{ github.token }}
+          tags: ${{env.tag_name}}
+
 
   compile:
     runs-on: ubuntu-latest
@@ -81,6 +102,15 @@ jobs:
     -
       name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v2
+      with:
+        driver-opts: ${{env.buildx_driver_opts}}
+    - 
+      name: Login to Docker registry
+      uses: docker/login-action@v2
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
     -
       name: Build
       uses: docker/build-push-action@v4
@@ -92,13 +122,10 @@ jobs:
         file: ./build/workflow.Dockerfile
         target: compile
         push: false
-        cache-to: |
-          type=gha,
-          mode=max
-        cache-from: |
-          type=gha,
-          mode=max
+        cache-to: ${{env.cache_to_params}}
+        cache-from: ${{env.cache_from_params}}
         github-token: ${{ github.token }}
+        tags: ${{env.tag_name}}
 
   core-unit-test:
     runs-on: ubuntu-latest
@@ -113,6 +140,15 @@ jobs:
       -
         name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
+        with:
+          driver-opts: ${{env.buildx_driver_opts}}
+      - 
+        name: Login to Docker registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
       -
         name: Build
         uses: docker/build-push-action@v4
@@ -121,11 +157,10 @@ jobs:
           file: ./build/workflow.Dockerfile
           target: test-core-unit
           push: false
-          cache-to: |
-            type=gha
-          cache-from: |
-            type=gha
+          cache-to:  ${{env.cache_to_params}}
+          cache-from: ${{env.cache_from_params}}
           github-token: ${{ github.token }}
+          tags: ${{env.tag_name}}
 
   core-integration-test:
     runs-on: ubuntu-latest
@@ -146,6 +181,15 @@ jobs:
       -
         name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
+        with:
+          driver-opts: ${{env.buildx_driver_opts}}
+      - 
+        name: Login to Docker registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
       -
         name: Build
         uses: docker/build-push-action@v4
@@ -154,11 +198,10 @@ jobs:
           file: ./build/workflow.Dockerfile
           target: test-core-integration
           push: false
-          cache-to: |
-            type=gha
-          cache-from: |
-            type=gha
+          cache-to:  ${{env.cache_to_params}}
+          cache-from: ${{env.cache_from_params}}
           github-token: ${{ github.token }}
+          tags: ${{env.tag_name}}
 
 
   # e2e:

CONTRIBUTING.md

@@ -338,6 +338,7 @@ Please refer to the [README](./docs/README.md) for more information and local te
 
 - **Code with variables**: Make sure that code snippets can be used by setting environment variables, instead of manually replacing a placeholder.
 - **Embedded files**: When embedding mdx files, make sure the template ist prefixed by "_" (lowdash). The content will be rendered inside the parent page, but is not accessible individually (eg, by search).
+- **Don't repeat yourself**: When using the same content in multiple places, save and manage the content as separate file and make use of embedded files to import it into other docs pages.
 - **Embedded code**: You can embed code snippets from a repository. See the [plugin](https://github.com/saucelabs/docusaurus-theme-github-codeblock#usage) for usage.
 
 ### Docs Pull Request

build/workflow.Dockerfile

@@ -102,7 +102,7 @@ RUN make assets
 # #######################################
 # download console dependencies
 # #######################################
-FROM node:alpine AS console-deps
+FROM node:18-buster AS console-deps
 
 WORKDIR /zitadel/console
 
@@ -114,7 +114,7 @@ RUN yarn install --frozen-lockfile
 # #######################################
 # generate console client
 # #######################################
-FROM node:alpine AS console-client
+FROM node:18-buster AS console-client
 
 WORKDIR /zitadel/console
 

cmd/setup/10.go

@@ -13,11 +13,11 @@ import (
 )
 
 var (
-	//go:embed 10_create_temp_table.sql
+	//go:embed 10/10_create_temp_table.sql
 	correctCreationDate10CreateTable string
-	//go:embed 10_fill_table.sql
+	//go:embed 10/10_fill_table.sql
 	correctCreationDate10FillTable string
-	//go:embed 10_update.sql
+	//go:embed 10/10_update.sql
 	correctCreationDate10Update string
 )
 

cmd/setup/11.go

@@ -0,0 +1,32 @@
+package setup
+
+import (
+	"context"
+	_ "embed"
+
+	"github.com/zitadel/zitadel/internal/database"
+)
+
+var (
+	//go:embed 11.sql
+	addEventCreatedAt string
+)
+
+type AddEventCreatedAt struct {
+	step10   *CorrectCreationDate
+	dbClient *database.DB
+}
+
+func (mig *AddEventCreatedAt) Execute(ctx context.Context) error {
+	// execute step 10 again because events created after the first execution of step 10
+	// could still have the wrong ordering of sequences and creation date
+	if err := mig.step10.Execute(ctx); err != nil {
+		return err
+	}
+	_, err := mig.dbClient.ExecContext(ctx, addEventCreatedAt)
+	return err
+}
+
+func (mig *AddEventCreatedAt) String() string {
+	return "11_event_created_at"
+}

cmd/setup/11.sql

@@ -0,0 +1,15 @@
+BEGIN;
+-- create table with empty created_at
+ALTER TABLE eventstore.events ADD COLUMN created_at TIMESTAMPTZ DEFAULT NULL;
+COMMIT;
+
+BEGIN;
+-- backfill created_at
+UPDATE eventstore.events SET created_at = creation_date WHERE created_at IS NULL;
+COMMIT;
+
+BEGIN;
+-- set column rules
+ALTER TABLE eventstore.events ALTER COLUMN created_at SET DEFAULT clock_timestamp();
+ALTER TABLE eventstore.events ALTER COLUMN created_at SET NOT NULL;
+COMMIT;

cmd/setup/config.go

@@ -66,6 +66,7 @@ type Steps struct {
 	s8AuthTokens         *AuthTokenIndexes
 	s9EventstoreIndexes2 *EventstoreIndexesNew
 	CorrectCreationDate  *CorrectCreationDate
+	s11AddEventCreatedAt *AddEventCreatedAt
 }
 
 type encryptionKeyConfig struct {

cmd/setup/setup.go

@@ -91,6 +91,7 @@ func Setup(config *Config, steps *Steps, masterKey string) {
 	steps.s8AuthTokens = &AuthTokenIndexes{dbClient: dbClient}
 	steps.s9EventstoreIndexes2 = New09(dbClient)
 	steps.CorrectCreationDate.dbClient = dbClient
+	steps.s11AddEventCreatedAt = &AddEventCreatedAt{dbClient: dbClient, step10: steps.CorrectCreationDate}
 
 	err = projection.Create(ctx, dbClient, eventstoreClient, config.Projections, nil, nil)
 	logging.OnError(err).Fatal("unable to start projections")
@@ -128,6 +129,8 @@ func Setup(config *Config, steps *Steps, masterKey string) {
 	logging.OnError(err).Fatal("unable to migrate step 9")
 	err = migration.Migrate(ctx, eventstoreClient, steps.CorrectCreationDate)
 	logging.OnError(err).Fatal("unable to migrate step 10")
+	err = migration.Migrate(ctx, eventstoreClient, steps.s11AddEventCreatedAt)
+	logging.OnError(err).Fatal("unable to migrate step 11")
 
 	for _, repeatableStep := range repeatableSteps {
 		err = migration.Migrate(ctx, eventstoreClient, repeatableStep)