MG-2235 - Check gRPC service is healthy during setup (#2245)

Signed-off-by: Rodney Osodo <28790446+rodneyosodo@users.noreply.github.com>
absmach · Jun 12, 2024 · 487dcc6 · 487dcc6
1 parent 75a66a4
commit 487dcc6
Show file tree

Hide file tree

Showing 25 changed files with 123 additions and 53 deletions.
diff --git a/auth/api/grpc/endpoint_test.go b/auth/api/grpc/endpoint_test.go
@@ -63,7 +63,7 @@ func startGRPCServer(svc auth.Service, port int) {
 }
 
 func TestIssue(t *testing.T) {
-	conn, err := grpc.Dial(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	conn, err := grpc.NewClient(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
 	assert.Nil(t, err, fmt.Sprintf("Unexpected error creating client connection %s", err))
 	client := grpcapi.NewClient(conn, time.Second)
 
@@ -132,7 +132,7 @@ func TestIssue(t *testing.T) {
 }
 
 func TestRefresh(t *testing.T) {
-	conn, err := grpc.Dial(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	conn, err := grpc.NewClient(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
 	assert.Nil(t, err, fmt.Sprintf("Unexpected error creating client connection %s", err))
 	client := grpcapi.NewClient(conn, time.Second)
 
@@ -178,7 +178,7 @@ func TestRefresh(t *testing.T) {
 }
 
 func TestIdentify(t *testing.T) {
-	conn, err := grpc.Dial(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	conn, err := grpc.NewClient(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
 	assert.Nil(t, err, fmt.Sprintf("Unexpected error creating client connection %s", err))
 	client := grpcapi.NewClient(conn, time.Second)
 
@@ -222,7 +222,7 @@ func TestIdentify(t *testing.T) {
 }
 
 func TestAuthorize(t *testing.T) {
-	conn, err := grpc.Dial(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	conn, err := grpc.NewClient(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
 	assert.Nil(t, err, fmt.Sprintf("Unexpected error creating client connection %s", err))
 	client := grpcapi.NewClient(conn, time.Second)
 
@@ -344,7 +344,7 @@ func TestAuthorize(t *testing.T) {
 }
 
 func TestAddPolicy(t *testing.T) {
-	conn, err := grpc.Dial(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	conn, err := grpc.NewClient(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
 	assert.Nil(t, err, fmt.Sprintf("Unexpected error creating client connection %s", err))
 	client := grpcapi.NewClient(conn, time.Second)
 
@@ -398,7 +398,7 @@ func TestAddPolicy(t *testing.T) {
 }
 
 func TestAddPolicies(t *testing.T) {
-	conn, err := grpc.Dial(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	conn, err := grpc.NewClient(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
 	assert.Nil(t, err, fmt.Sprintf("Unexpected error creating client connection %s", err))
 	client := grpcapi.NewClient(conn, time.Second)
 
@@ -460,7 +460,7 @@ func TestAddPolicies(t *testing.T) {
 }
 
 func TestDeletePolicy(t *testing.T) {
-	conn, err := grpc.Dial(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	conn, err := grpc.NewClient(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
 	assert.Nil(t, err, fmt.Sprintf("Unexpected error creating client connection %s", err))
 	client := grpcapi.NewClient(conn, time.Second)
 
@@ -513,7 +513,7 @@ func TestDeletePolicy(t *testing.T) {
 }
 
 func TestDeletePolicies(t *testing.T) {
-	conn, err := grpc.Dial(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	conn, err := grpc.NewClient(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
 	assert.Nil(t, err, fmt.Sprintf("Unexpected error creating client connection %s", err))
 	client := grpcapi.NewClient(conn, time.Second)
 
@@ -576,7 +576,7 @@ func TestDeletePolicies(t *testing.T) {
 }
 
 func TestListObjects(t *testing.T) {
-	conn, err := grpc.Dial(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	conn, err := grpc.NewClient(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
 	assert.Nil(t, err, fmt.Sprintf("Unexpected error creating client connection %s", err))
 	client := grpcapi.NewClient(conn, time.Second)
 
@@ -626,7 +626,7 @@ func TestListObjects(t *testing.T) {
 }
 
 func TestListAllObjects(t *testing.T) {
-	conn, err := grpc.Dial(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	conn, err := grpc.NewClient(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
 	assert.Nil(t, err, fmt.Sprintf("Unexpected error creating client connection %s", err))
 	client := grpcapi.NewClient(conn, time.Second)
 
@@ -676,7 +676,7 @@ func TestListAllObjects(t *testing.T) {
 }
 
 func TestCountObects(t *testing.T) {
-	conn, err := grpc.Dial(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	conn, err := grpc.NewClient(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
 	assert.Nil(t, err, fmt.Sprintf("Unexpected error creating client connection %s", err))
 	client := grpcapi.NewClient(conn, time.Second)
 
@@ -726,7 +726,7 @@ func TestCountObects(t *testing.T) {
 }
 
 func TestListSubjects(t *testing.T) {
-	conn, err := grpc.Dial(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	conn, err := grpc.NewClient(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
 	assert.Nil(t, err, fmt.Sprintf("Unexpected error creating client connection %s", err))
 	client := grpcapi.NewClient(conn, time.Second)
 
@@ -776,7 +776,7 @@ func TestListSubjects(t *testing.T) {
 }
 
 func TestListAllSubjects(t *testing.T) {
-	conn, err := grpc.Dial(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	conn, err := grpc.NewClient(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
 	assert.Nil(t, err, fmt.Sprintf(`"Unexpected error creating client connection %s"`, err))
 	client := grpcapi.NewClient(conn, time.Second)
 
@@ -826,7 +826,7 @@ func TestListAllSubjects(t *testing.T) {
 }
 
 func TestCountSubjects(t *testing.T) {
-	conn, err := grpc.Dial(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	conn, err := grpc.NewClient(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
 	assert.Nil(t, err, fmt.Sprintf("Unexpected error creating client connection %s", err))
 	client := grpcapi.NewClient(conn, time.Second)
 
@@ -881,7 +881,7 @@ func TestCountSubjects(t *testing.T) {
 }
 
 func TestListPermissions(t *testing.T) {
-	conn, err := grpc.Dial(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	conn, err := grpc.NewClient(authAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
 	assert.Nil(t, err, fmt.Sprintf("Unexpected error creating client connection %s", err))
 	client := grpcapi.NewClient(conn, time.Second)
 

diff --git a/cmd/bootstrap/main.go b/cmd/bootstrap/main.go
@@ -107,7 +107,7 @@ func main() {
 		return
 	}
 
-	authClient, authHandler, err := auth.Setup(authConfig)
+	authClient, authHandler, err := auth.Setup(ctx, authConfig)
 	if err != nil {
 		logger.Error(err.Error())
 		exitCode = 1

diff --git a/cmd/cassandra-reader/main.go b/cmd/cassandra-reader/main.go
@@ -76,7 +76,7 @@ func main() {
 		return
 	}
 
-	ac, acHandler, err := auth.Setup(authConfig)
+	ac, acHandler, err := auth.Setup(ctx, authConfig)
 	if err != nil {
 		logger.Error(err.Error())
 		exitCode = 1
@@ -93,7 +93,7 @@ func main() {
 		return
 	}
 
-	tc, tcHandler, err := auth.SetupAuthz(authConfig)
+	tc, tcHandler, err := auth.SetupAuthz(ctx, authConfig)
 	if err != nil {
 		logger.Error(err.Error())
 		exitCode = 1

diff --git a/cmd/certs/main.go b/cmd/certs/main.go
@@ -126,7 +126,7 @@ func main() {
 		return
 	}
 
-	authClient, authHandler, err := auth.Setup(authConfig)
+	authClient, authHandler, err := auth.Setup(ctx, authConfig)
 	if err != nil {
 		logger.Error(err.Error())
 		exitCode = 1

diff --git a/cmd/coap/main.go b/cmd/coap/main.go
@@ -94,7 +94,7 @@ func main() {
 		return
 	}
 
-	authClient, authHandler, err := auth.SetupAuthz(authConfig)
+	authClient, authHandler, err := auth.SetupAuthz(ctx, authConfig)
 	if err != nil {
 		logger.Error(err.Error())
 		exitCode = 1

diff --git a/cmd/http/main.go b/cmd/http/main.go
@@ -94,7 +94,7 @@ func main() {
 		return
 	}
 
-	authClient, authHandler, err := auth.SetupAuthz(authConfig)
+	authClient, authHandler, err := auth.SetupAuthz(ctx, authConfig)
 	if err != nil {
 		logger.Error(err.Error())
 		exitCode = 1

diff --git a/cmd/influxdb-reader/main.go b/cmd/influxdb-reader/main.go
@@ -75,7 +75,7 @@ func main() {
 		return
 	}
 
-	ac, acHandler, err := auth.Setup(authConfig)
+	ac, acHandler, err := auth.Setup(ctx, authConfig)
 	if err != nil {
 		logger.Error(err.Error())
 		exitCode = 1
@@ -92,7 +92,7 @@ func main() {
 		return
 	}
 
-	tc, tcHandler, err := auth.SetupAuthz(authConfig)
+	tc, tcHandler, err := auth.SetupAuthz(ctx, authConfig)
 	if err != nil {
 		logger.Error(err.Error())
 		exitCode = 1

diff --git a/cmd/invitations/main.go b/cmd/invitations/main.go
@@ -98,7 +98,7 @@ func main() {
 		exitCode = 1
 		return
 	}
-	authClient, authHandler, err := auth.Setup(authConfig)
+	authClient, authHandler, err := auth.Setup(ctx, authConfig)
 	if err != nil {
 		logger.Error(err.Error())
 		exitCode = 1

diff --git a/cmd/mongodb-reader/main.go b/cmd/mongodb-reader/main.go
@@ -84,7 +84,7 @@ func main() {
 		return
 	}
 
-	ac, acHandler, err := auth.Setup(authConfig)
+	ac, acHandler, err := auth.Setup(ctx, authConfig)
 	if err != nil {
 		logger.Error(err.Error())
 		exitCode = 1
@@ -101,7 +101,7 @@ func main() {
 		return
 	}
 
-	tc, tcHandler, err := auth.SetupAuthz(authConfig)
+	tc, tcHandler, err := auth.SetupAuthz(ctx, authConfig)
 	if err != nil {
 		logger.Error(err.Error())
 		exitCode = 1

diff --git a/cmd/mqtt/main.go b/cmd/mqtt/main.go
@@ -171,7 +171,7 @@ func main() {
 		return
 	}
 
-	authClient, authHandler, err := auth.SetupAuthz(authConfig)
+	authClient, authHandler, err := auth.SetupAuthz(ctx, authConfig)
 	if err != nil {
 		logger.Error(err.Error())
 		exitCode = 1

diff --git a/cmd/postgres-reader/main.go b/cmd/postgres-reader/main.go
@@ -90,7 +90,7 @@ func main() {
 		return
 	}
 
-	ac, acHandler, err := auth.Setup(authConfig)
+	ac, acHandler, err := auth.Setup(ctx, authConfig)
 	if err != nil {
 		logger.Error(err.Error())
 		exitCode = 1
@@ -107,7 +107,7 @@ func main() {
 		return
 	}
 
-	tc, tcHandler, err := auth.SetupAuthz(authConfig)
+	tc, tcHandler, err := auth.SetupAuthz(ctx, authConfig)
 	if err != nil {
 		logger.Error(err.Error())
 		exitCode = 1

diff --git a/cmd/smpp-notifier/main.go b/cmd/smpp-notifier/main.go
@@ -139,7 +139,7 @@ func main() {
 		return
 	}
 
-	authClient, authHandler, err := auth.Setup(authConfig)
+	authClient, authHandler, err := auth.Setup(ctx, authConfig)
 	if err != nil {
 		logger.Error(err.Error())
 		exitCode = 1

diff --git a/cmd/smtp-notifier/main.go b/cmd/smtp-notifier/main.go
@@ -140,7 +140,7 @@ func main() {
 		return
 	}
 
-	authClient, authHandler, err := auth.Setup(authConfig)
+	authClient, authHandler, err := auth.Setup(ctx, authConfig)
 	if err != nil {
 		logger.Error(err.Error())
 		exitCode = 1

diff --git a/cmd/things/main.go b/cmd/things/main.go
@@ -158,7 +158,7 @@ func main() {
 			return
 		}
 
-		authServiceClient, authHandler, err := auth.Setup(authConfig)
+		authServiceClient, authHandler, err := auth.Setup(ctx, authConfig)
 		if err != nil {
 			logger.Error(err.Error())
 			exitCode = 1

diff --git a/cmd/timescale-reader/main.go b/cmd/timescale-reader/main.go
@@ -90,7 +90,7 @@ func main() {
 		return
 	}
 
-	ac, acHandler, err := auth.Setup(authConfig)
+	ac, acHandler, err := auth.Setup(ctx, authConfig)
 	if err != nil {
 		logger.Error(err.Error())
 		exitCode = 1
@@ -107,7 +107,7 @@ func main() {
 		return
 	}
 
-	tc, tcHandler, err := auth.SetupAuthz(authConfig)
+	tc, tcHandler, err := auth.SetupAuthz(ctx, authConfig)
 	if err != nil {
 		logger.Error(err.Error())
 		exitCode = 1

diff --git a/cmd/twins/main.go b/cmd/twins/main.go
@@ -134,7 +134,7 @@ func main() {
 			return
 		}
 
-		authServiceClient, authHandler, err := auth.Setup(authConfig)
+		authServiceClient, authHandler, err := auth.Setup(ctx, authConfig)
 		if err != nil {
 			logger.Error(err.Error())
 			exitCode = 1

diff --git a/cmd/users/main.go b/cmd/users/main.go
@@ -154,7 +154,7 @@ func main() {
 		return
 	}
 
-	authClient, authHandler, err := auth.Setup(authConfig)
+	authClient, authHandler, err := auth.Setup(ctx, authConfig)
 	if err != nil {
 		logger.Error(err.Error())
 		exitCode = 1

diff --git a/cmd/ws/main.go b/cmd/ws/main.go
@@ -96,7 +96,7 @@ func main() {
 		return
 	}
 
-	authClient, authHandler, err := auth.SetupAuthz(authConfig)
+	authClient, authHandler, err := auth.SetupAuthz(ctx, authConfig)
 	if err != nil {
 		logger.Error(err.Error())
 		exitCode = 1

diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
@@ -175,6 +175,7 @@ services:
     container_name: magistrala-invitations
     restart: on-failure
     depends_on:
+      - auth
       - invitations-db
     environment:
       MG_INVITATIONS_LOG_LEVEL: ${MG_INVITATIONS_LOG_LEVEL}
@@ -255,6 +256,7 @@ services:
     env_file:
       - .env
     depends_on:
+      - auth
       - things
       - users
       - mqtt-adapter

diff --git a/internal/server/grpc/grpc.go b/internal/server/grpc/grpc.go
@@ -18,16 +18,17 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/health"
+	grpchealth "google.golang.org/grpc/health/grpc_health_v1"
 )
 
-const (
-	stopWaitTime = 5 * time.Second
-)
+const stopWaitTime = 5 * time.Second
 
 type Server struct {
 	server.BaseServer
 	server          *grpc.Server
 	registerService serviceRegister
+	health          *health.Server
 }
 
 type serviceRegister func(srv *grpc.Server)
@@ -116,7 +117,10 @@ func (s *Server) Start() error {
 	grpcServerOptions = append(grpcServerOptions, creds)
 
 	s.server = grpc.NewServer(grpcServerOptions...)
+	s.health = health.NewServer()
+	grpchealth.RegisterHealthServer(s.server, s.health)
 	s.registerService(s.server)
+	s.health.SetServingStatus(s.Name, grpchealth.HealthCheckResponse_SERVING)
 
 	go func() {
 		errCh <- s.server.Serve(listener)
@@ -136,6 +140,7 @@ func (s *Server) Stop() error {
 	c := make(chan bool)
 	go func() {
 		defer close(c)
+		s.health.Shutdown()
 		s.server.GracefulStop()
 	}()
 	select {