tie loose ends, documentation

.github/workflows/integration.yml

@@ -9,7 +9,7 @@ on:
       - '**'
 
 jobs:
-  integration-tests:
+  run:
     strategy:
       matrix:
         db: [cockroach, postgres]
@@ -36,7 +36,7 @@ jobs:
       - name: Download Go modules
         run: go mod download
       - name: Start ${{ matrix.db }} database
-        run: docker compose -f e2e/config/integration/docker-compose.yaml up --wait ${INTEGRATION_DB_FLAVOR}
+        run: docker compose -f internal/integration/config/docker-compose.yaml up --wait ${INTEGRATION_DB_FLAVOR}
       - name: Run zitadel init and setup
         run: |
           go run main.go init --config internal/integration/config/zitadel.yaml --config internal/integration/config/${INTEGRATION_DB_FLAVOR}.yaml

CONTRIBUTING.md

@@ -199,6 +199,21 @@ When you are happy with your changes, you can cleanup your environment.
 docker compose --file ./e2e/config/host.docker.internal/docker-compose.yaml down
 ```
 
+#### Integration tests
+
+In order to run the integrations tests for the gRPC API, PostgreSQL and CockroachDB must be running and initialized.
+
+```bash
+export INTEGRATION_DB_FLAVOR="cockroach"
+docker compose -f internal/integration/config/docker-compose.yaml up --wait ${INTEGRATION_DB_FLAVOR}
+go run main.go init --config internal/integration/config/zitadel.yaml --config internal/integration/config/${INTEGRATION_DB_FLAVOR}.yaml
+go run main.go setup --masterkey MasterkeyNeedsToHave32Characters --config internal/integration/config/zitadel.yaml --config internal/integration/config/${INTEGRATION_DB_FLAVOR}.yaml
+go test -tags=integration -race -parallel 1 ./internal/integration ./internal/api/grpc/...
+docker compose -f internal/integration/config/docker-compose.yaml down
+```
+
+The above can be repeated with `INTEGRATION_DB_FLAVOR="postgres"`.
+
 ### Console
 
 By executing the commands from this section, you run everything you need to develop the console locally.

cmd/start/start.go

@@ -21,6 +21,7 @@ import (
 	"github.com/zitadel/saml/pkg/provider"
 	"golang.org/x/net/http2"
 	"golang.org/x/net/http2/h2c"
+	"golang.org/x/sys/unix"
 
 	"github.com/zitadel/zitadel/cmd/key"
 	cmd_tls "github.com/zitadel/zitadel/cmd/tls"
@@ -341,10 +342,21 @@ func startAPIs(
 	return nil
 }
 
+func reusePort(network, address string, conn syscall.RawConn) error {
+	return conn.Control(func(descriptor uintptr) {
+		err := syscall.SetsockoptInt(int(descriptor), syscall.SOL_SOCKET, unix.SO_REUSEPORT, 1)
+		if err != nil {
+			panic(err)
+		}
+	})
+}
+
 func listen(ctx context.Context, router *mux.Router, port uint16, tlsConfig *tls.Config, shutdown <-chan os.Signal) error {
 	http2Server := &http2.Server{}
 	http1Server := &http.Server{Handler: h2c.NewHandler(router, http2Server), TLSConfig: tlsConfig}
-	lis, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
+
+	lc := &net.ListenConfig{Control: reusePort}
+	lis, err := lc.Listen(ctx, "tcp", fmt.Sprintf(":%d", port))
 	if err != nil {
 		return fmt.Errorf("tcp listener on %d failed: %w", port, err)
 	}

internal/api/grpc/user/v2/user_integration_test.go

@@ -57,7 +57,7 @@ func TestServer_AddHumanUser(t *testing.T) {
 				&user.AddHumanUserRequest{
 					Organisation: &object.Organisation{
 						Org: &object.Organisation_OrgId{
-							OrgId: "211137963315232910",
+							OrgId: Tester.Organisation.ID,
 						},
 					},
 					Profile: &user.SetHumanProfile{
@@ -97,7 +97,7 @@ func TestServer_AddHumanUser(t *testing.T) {
 				&user.AddHumanUserRequest{
 					Organisation: &object.Organisation{
 						Org: &object.Organisation_OrgId{
-							OrgId: "211137963315232910",
+							OrgId: Tester.Organisation.ID,
 						},
 					},
 					Profile: &user.SetHumanProfile{
@@ -142,7 +142,7 @@ func TestServer_AddHumanUser(t *testing.T) {
 				&user.AddHumanUserRequest{
 					Organisation: &object.Organisation{
 						Org: &object.Organisation_OrgId{
-							OrgId: "211137963315232910",
+							OrgId: Tester.Organisation.ID,
 						},
 					},
 					Profile: &user.SetHumanProfile{
@@ -188,7 +188,7 @@ func TestServer_AddHumanUser(t *testing.T) {
 				&user.AddHumanUserRequest{
 					Organisation: &object.Organisation{
 						Org: &object.Organisation_OrgId{
-							OrgId: "211137963315232910",
+							OrgId: Tester.Organisation.ID,
 						},
 					},
 					Profile: &user.SetHumanProfile{
@@ -229,7 +229,7 @@ func TestServer_AddHumanUser(t *testing.T) {
 				&user.AddHumanUserRequest{
 					Organisation: &object.Organisation{
 						Org: &object.Organisation_OrgId{
-							OrgId: "211137963315232910",
+							OrgId: Tester.Organisation.ID,
 						},
 					},
 					Email: &user.SetHumanEmail{
@@ -260,7 +260,7 @@ func TestServer_AddHumanUser(t *testing.T) {
 				&user.AddHumanUserRequest{
 					Organisation: &object.Organisation{
 						Org: &object.Organisation_OrgId{
-							OrgId: "211137963315232910",
+							OrgId: Tester.Organisation.ID,
 						},
 					},
 					Profile: &user.SetHumanProfile{

internal/integration/assert.go

@@ -13,13 +13,29 @@ type DetailsMsg interface {
 	GetDetails() *object.Details
 }
 
+// AssertDetails asserts values in a message's object Details,
+// if the object Details in expected is a non-nil value.
+// It targets API v2 messages that have the `GetDetails()` method.
+//
+// Dynamically generated values are not compared with expected.
+// Instead a sanity check is performed.
+// For the sequence a non-zero value is expected.
+// The change date has to be now, with a tollerance of 1 second.
+//
+// The resource owner is compared with expected and is
+// therefore the only value that has to be set.
 func AssertDetails[D DetailsMsg](t testing.TB, exptected, actual D) {
 	wantDetails, gotDetails := exptected.GetDetails(), actual.GetDetails()
-
-	if wantDetails != nil {
-		assert.NotZero(t, gotDetails.GetSequence())
+	if wantDetails == nil {
+		assert.Nil(t, gotDetails)
+		return
 	}
-	wantCD, gotCD := wantDetails.GetChangeDate().AsTime(), gotDetails.GetChangeDate().AsTime()
-	assert.WithinRange(t, gotCD, wantCD, wantCD.Add(time.Minute))
+
+	assert.NotZero(t, gotDetails.GetSequence())
+
+	gotCD := gotDetails.GetChangeDate().AsTime()
+	now := time.Now()
+	assert.WithinRange(t, gotCD, now.Add(-time.Second), now.Add(time.Second))
+
 	assert.Equal(t, wantDetails.GetResourceOwner(), gotDetails.GetResourceOwner())
 }

internal/integration/assert_test.go

@@ -0,0 +1,51 @@
+package integration
+
+import (
+	"testing"
+
+	"google.golang.org/protobuf/types/known/timestamppb"
+
+	object "github.com/zitadel/zitadel/pkg/grpc/object/v2alpha"
+)
+
+type myMsg struct {
+	details *object.Details
+}
+
+func (m myMsg) GetDetails() *object.Details {
+	return m.details
+}
+
+func TestAssertDetails(t *testing.T) {
+	tests := []struct {
+		name      string
+		exptected myMsg
+		actual    myMsg
+	}{
+		{
+			name:      "nil",
+			exptected: myMsg{},
+			actual:    myMsg{},
+		},
+		{
+			name: "values",
+			exptected: myMsg{
+				details: &object.Details{
+					ResourceOwner: "me",
+				},
+			},
+			actual: myMsg{
+				details: &object.Details{
+					Sequence:      123,
+					ChangeDate:    timestamppb.Now(),
+					ResourceOwner: "me",
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			AssertDetails(t, tt.exptected, tt.actual)
+		})
+	}
+}

e2e/config/integration/docker-compose.yaml → internal/integration/config/docker-compose.yaml

@@ -3,7 +3,7 @@ version: '3.8'
 services:
   cockroach:
     extends:
-      file: '../localhost/docker-compose.yaml'
+      file: '../../../e2e/config/localhost/docker-compose.yaml'
       service: 'db'
 
   postgres:

internal/integration/integration.go

@@ -41,6 +41,11 @@ var (
 	postgresYAML []byte
 )
 
+// UserType provides constants that give
+// a short explinanation with the purpose
+// a serverice user.
+// This allows to pre-create users with
+// different permissions and reuse them.
 type UserType int
 
 //go:generate stringer -type=UserType
@@ -49,11 +54,13 @@ const (
 	OrgOwner
 )
 
+// User information with a Personal Access Token.
 type User struct {
 	*query.User
 	Token string
 }
 
+// Tester is a Zitadel server and client with all resources available for testing.
 type Tester struct {
 	*start.Server
 
@@ -113,7 +120,7 @@ func (s *Tester) pollHealth(ctx context.Context) (err error) {
 }
 
 const (
-	SystemUser = "integration1"
+	SystemUser = "integration"
 )
 
 func (s *Tester) createSystemUser(ctx context.Context) {
@@ -169,6 +176,7 @@ func (s *Tester) WithSystemAuthorization(ctx context.Context, u UserType) contex
 	return metadata.AppendToOutgoingContext(ctx, "Authorization", fmt.Sprintf("Bearer %s", s.Users[u].Token))
 }
 
+// Done send an interrupt signal to cleanly shutdown the server.
 func (s *Tester) Done() {
 	err := s.GRPCClientConn.Close()
 	logging.OnError(err).Error("integration tester client close")
@@ -177,6 +185,20 @@ func (s *Tester) Done() {
 	s.wg.Wait()
 }
 
+// NewTester start a new Zitadel server by passing the default commandline.
+// The server will listen on the configured port.
+// The database configuration that will be used can be set by the
+// INTEGRATION_DB_FLAVOR environment variable and can have the values "cockroach"
+// or "postgres". Defaults to "cockroach".
+//
+// The deault Instance and Organisation are read from the DB and system
+// users are created as needed.
+//
+// After the server is started, a [grpc.ClientConn] will be created and
+// the server is polled for it's health status.
+//
+// Note: the database must already be setup and intialized before
+// using NewTester. See the CONTRIBUTING.md document for details.
 func NewTester(ctx context.Context) *Tester {
 	args := strings.Split(commandLine, " ")