Fix potential race condition in boulder-ca shutdown (#5277)

`cai.Stop()` called from boulder-ca could potentially exit before errors
emitted by `caSrv` and `ocspSrv` are logged. This could lead to
boulder-ca erroneously exiting `0` instead of `1`.

Add a `sync.WaitGroup`. Increment the waitgroup before `caSrv.Serve()`
and `ocspSrv.Serv()` are spun off. Await the waitgroup before
`cai.Stop()` is called.

Fixes #5246

cmd/boulder-ca/main.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"sync"
 
 	"github.com/beeker1121/goque"
 	cfsslConfig "github.com/cloudflare/cfssl/config"
@@ -339,15 +340,18 @@ func main() {
 	go cai.LogOCSPLoop()
 
 	serverMetrics := bgrpc.NewServerMetrics(scope)
+	var wg sync.WaitGroup
 
 	caSrv, caListener, err := bgrpc.NewServer(c.CA.GRPCCA, tlsConfig, serverMetrics, clk)
 	cmd.FailOnError(err, "Unable to setup CA gRPC server")
 	caWrapper := bgrpc.NewCertificateAuthorityServer(cai)
 	capb.RegisterCertificateAuthorityServer(caSrv, caWrapper)
 	caHealth := health.NewServer()
 	healthpb.RegisterHealthServer(caSrv, caHealth)
+	wg.Add(1)
 	go func() {
 		cmd.FailOnError(cmd.FilterShutdownErrors(caSrv.Serve(caListener)), "CA gRPC service failed")
+		wg.Done()
 	}()
 
 	ocspSrv, ocspListener, err := bgrpc.NewServer(c.CA.GRPCOCSPGenerator, tlsConfig, serverMetrics, clk)
@@ -356,16 +360,19 @@ func main() {
 	capb.RegisterOCSPGeneratorServer(ocspSrv, ocspWrapper)
 	ocspHealth := health.NewServer()
 	healthpb.RegisterHealthServer(ocspSrv, ocspHealth)
+	wg.Add(1)
 	go func() {
 		cmd.FailOnError(cmd.FilterShutdownErrors(ocspSrv.Serve(ocspListener)),
 			"OCSPGenerator gRPC service failed")
+		wg.Done()
 	}()
 
 	go cmd.CatchSignals(logger, func() {
 		caHealth.Shutdown()
 		ocspHealth.Shutdown()
 		caSrv.GracefulStop()
 		ocspSrv.GracefulStop()
+		wg.Wait()
 		cai.Stop()
 	})