gRPC Liveness Probe (#200)

pkg/ensign/server.go

@@ -16,6 +16,7 @@ import (
 	"github.com/rotationalio/ensign/pkg/ensign/interceptors"
 	"github.com/rotationalio/ensign/pkg/ensign/o11y"
 	"github.com/rotationalio/ensign/pkg/utils/logger"
+	health "github.com/rotationalio/ensign/pkg/utils/probez/grpc/v1"
 	"github.com/rotationalio/ensign/pkg/utils/sentry"
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
@@ -41,7 +42,9 @@ func init() {
 
 // An Ensign server implements the Ensign service as defined by the wire protocol.
 type Server struct {
+	health.ProbeServer
 	api.UnimplementedEnsignServer
+
 	srv     *grpc.Server  // The gRPC server that handles incoming requests in individual go routines
 	conf    config.Config // Primary source of truth for server configuration
 	started time.Time     // The timestamp that the server was started (for uptime)
@@ -104,6 +107,9 @@ func (s *Server) Serve() (err error) {
 		s.echan <- s.Shutdown()
 	}()
 
+	// Set the server to a not serving state
+	s.NotHealthy()
+
 	// Run monitoring and metrics server
 	if err = o11y.Serve(s.conf.Monitoring); err != nil {
 		log.Error().Err(err).Msg("could not start monitoring server")
@@ -129,6 +135,9 @@ func (s *Server) Serve() (err error) {
 	// Now that the server is running set the start time to track uptime
 	s.started = time.Now()
 
+	// Set the server to ready and serving requests
+	s.Healthy()
+
 	// Listen for any fatal errors on the error channel, blocking while the server go
 	// routine does its work. If the error is nil we expect a graceful shutdown.
 	if err = <-s.echan; err != nil {
@@ -151,6 +160,9 @@ func (s *Server) Run(sock net.Listener) {
 // return a multierror if there were multiple problems during shutdown but it will
 // attempt to close all open services and processes.
 func (s *Server) Shutdown() (err error) {
+	// Set the server to a not serving state
+	s.NotHealthy()
+
 	errs := make([]error, 0)
 	log.Info().Msg("gracefully shutting down ensign server")
 	s.srv.GracefulStop()

pkg/utils/probez/generate.go

@@ -0,0 +1,3 @@
+package probez
+
+//go:generate protoc -I=$GOPATH/src/github.com/rotationalio/ensign/proto --go_out=. --go_opt=module=github.com/rotationalio/ensign/pkg/utils/probez --go-grpc_out=. --go-grpc_opt=module=github.com/rotationalio/ensign/pkg/utils/probez grpc/health/v1/health.proto