Metadata service: refresh Metatron certs every 5 minutes

cmd/titus-metadata-service/main.go

@@ -30,6 +30,8 @@ import (
 // 169 is the first octet of 169.254...
 const defaultListeningPort = 8169
 
+const certRefreshTime = 5 * time.Minute
+
 /* Either returns a listener, or logs a fatal error */
 func getListener(listenPort int, listenerFd int64) net.Listener {
 	if listenerFd != -1 && listenPort != defaultListeningPort {
@@ -78,6 +80,20 @@ func readTaskConfigFile(taskID string) (*titus.ContainerInfo, error) {
 	return &cInfo, nil
 }
 
+func reloadSigner(ms *metadataserver.MetadataServer) {
+	t := time.NewTicker(certRefreshTime)
+	defer t.Stop()
+	for range t.C {
+		if newSigner, err := identity.NewDefaultSigner(); err != nil {
+			log.WithError(err).Fatal("Cannot instantiate new default signer")
+		} else {
+			if err := ms.SetSigner(newSigner); err != nil {
+				log.WithError(err).Error("Error reloading signing certificate")
+			}
+		}
+	}
+}
+
 func main() {
 	app := cli.NewApp()
 	app.Name = "titus-metadata-service"
@@ -236,6 +252,7 @@ func main() {
 		}
 		ms := metadataserver.NewMetaDataServer(context.Background(), mdscfg)
 		go notifySystemd()
+		go reloadSigner(ms)
 		// TODO: Wire up logic to shut down mds on signal
 		if err := http.Serve(listener, ms); err != nil {
 			return cli.NewExitError(err.Error(), 1)

metadataserver/identity.go

@@ -1,12 +1,15 @@
 package metadataserver
 
 import (
+	"crypto/x509"
 	"encoding/json"
 	"net/http"
 	"time"
 
 	"github.com/Netflix/titus-executor/api/netflix/titus"
+	"github.com/Netflix/titus-executor/metadataserver/identity"
 	"github.com/Netflix/titus-executor/metadataserver/metrics"
+	"github.com/pkg/errors"
 
 	"github.com/golang/protobuf/proto"
 	log "github.com/sirupsen/logrus"
@@ -29,6 +32,38 @@ func (ms *MetadataServer) generateTaskIdentity() *titus.TaskIdentity {
 	}
 }
 
+// SetSigner updates the identity server's signer
+func (ms *MetadataServer) SetSigner(newSigner *identity.Signer) error {
+	newCert, err := x509.ParseCertificate(newSigner.Certificate.Certificate[0])
+	if err != nil {
+		return errors.Wrap(err, "error parsing new certificate")
+	}
+
+	ms.signLock.Lock()
+	defer ms.signLock.Unlock()
+	oldSigner := ms.signer
+	oldCert, err := x509.ParseCertificate(oldSigner.Certificate.Certificate[0])
+	if err != nil {
+		return errors.Wrap(err, "error parsing old certificate")
+	}
+
+	log.WithFields(log.Fields{
+		"newSubject":      newCert.Subject,
+		"newIssuer":       newCert.Issuer,
+		"newNotBefore":    newCert.NotBefore,
+		"newNotAfter":     newCert.NotAfter,
+		"newSerialNumber": newCert.SerialNumber,
+		"oldSubject":      oldCert.Subject,
+		"oldIssuer":       oldCert.Issuer,
+		"oldNotBefore":    oldCert.NotBefore,
+		"oldNotAfter":     oldCert.NotAfter,
+		"oldSerialNumber": oldCert.SerialNumber,
+	}).Info("signer cert updated")
+
+	ms.signer = newSigner
+	return nil
+}
+
 func (ms *MetadataServer) taskIdentity(w http.ResponseWriter, r *http.Request) {
 	metrics.PublishIncrementCounter("handler.task-identity.count")
 
@@ -42,7 +77,9 @@ func (ms *MetadataServer) taskIdentity(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	ms.signLock.Lock()
 	sig, err := ms.signer.Sign(identData)
+	ms.signLock.Unlock()
 	if err != nil {
 		log.WithError(err).Error("Error signing data")
 		w.WriteHeader(http.StatusInternalServerError)
@@ -80,7 +117,9 @@ func (ms *MetadataServer) taskIdentityJSON(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
+	ms.signLock.Lock()
 	sig, err := ms.signer.SignString(identData)
+	ms.signLock.Unlock()
 	if err != nil {
 		log.WithError(err).Error("Error signing data")
 		w.WriteHeader(http.StatusInternalServerError)

metadataserver/server.go

@@ -8,6 +8,7 @@ import (
 	"net/http/httputil"
 	"net/url"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/Netflix/titus-executor/api/netflix/titus"
@@ -57,6 +58,8 @@ type MetadataServer struct {
 	eniID               string
 	container           *titus.ContainerInfo
 	signer              *identity.Signer
+	// Need to hold `signLock` while accessing `signer`
+	signLock sync.Mutex
 }
 
 func dumpRoutes(r *mux.Router) {