Merge pull request #212 from Netflix/run-metatron-in-container

Run certificate refresh periodically inside containers

.circleci/config.yml

@@ -38,6 +38,7 @@ jobs:
       - run: ./build.sh ignore-signals
       - run: ./build.sh pty
       - run: ./build.sh ubuntu-env-label
+      - run: BUILD_FROM_CHECKOUT_ROOT=true ./build.sh metatron
   ci-builder:
     docker:
       - image: docker:17.05.0-ce-git

cmd/titus-metadata-service/main.go

@@ -2,15 +2,22 @@ package main
 
 import (
 	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
 	"net"
 	"net/http"
 	"os"
+	"path/filepath"
 	"strconv"
 	"syscall"
 	"time"
 
+	"github.com/Netflix/titus-executor/api/netflix/titus"
+	runtimeTypes "github.com/Netflix/titus-executor/executor/runtime/types"
 	"github.com/Netflix/titus-executor/logsutil"
 	"github.com/Netflix/titus-executor/metadataserver"
+	"github.com/Netflix/titus-executor/metadataserver/identity"
 	"github.com/Netflix/titus-executor/metadataserver/types"
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
@@ -51,13 +58,31 @@ func makeFDListener(fd int64) net.Listener {
 	return l
 }
 
+func readTaskConfigFile(taskID string) (*titus.ContainerInfo, error) {
+	confFile := filepath.Join(runtimeTypes.TitusEnvironmentsDir, fmt.Sprintf("%s.json", taskID))
+	contents, err := ioutil.ReadFile(confFile) // nolint: gosec
+	if err != nil {
+		log.WithError(err).Errorf("Error reading file %s", confFile)
+		return nil, err
+	}
+
+	var cInfo titus.ContainerInfo
+	if err = json.Unmarshal(contents, &cInfo); err != nil {
+		log.WithError(err).Errorf("Error parsing JSON in file %s", confFile)
+		return nil, err
+	}
+
+	return &cInfo, nil
+}
+
 func main() {
 	app := cli.NewApp()
 	app.Name = "titus-metadata-service"
 	var listenerFd int64
 	var listenPort int
 	var debug bool
 	var backingMetadataServer string
+	var metatronEnabled bool
 	var optimistic bool
 	var region string
 	var iamARN string
@@ -115,6 +140,12 @@ func main() {
 			EnvVar:      "EC2_LOCAL_IPV4",
 			Destination: &ipv4Address,
 		},
+		cli.BoolFlag{
+			Name:        "metatron",
+			Usage:       "If set to true, the server will load certificates and use them to sign task identity documents",
+			EnvVar:      types.TitusMetatronVariableName,
+			Destination: &metatronEnabled,
+		},
 	}
 	app.Action = func(c *cli.Context) error {
 		if debug {
@@ -126,7 +157,21 @@ func main() {
 
 		/* Get the requisite configuration from environment variables */
 		listener := getListener(listenPort, listenerFd)
-		ms := metadataserver.NewMetaDataServer(context.Background(), backingMetadataServer, iamARN, titusTaskInstanceID, ipv4Address, region, optimistic)
+
+		var err error
+		var container *titus.ContainerInfo
+		var signer *identity.Signer
+
+		if container, err = readTaskConfigFile(titusTaskInstanceID); err != nil {
+			log.Fatal(err)
+		}
+		if metatronEnabled {
+			log.Info("Metatron enabled!")
+			if signer, err = identity.NewDefaultSigner(); err != nil {
+				log.Fatal(err)
+			}
+		}
+		ms := metadataserver.NewMetaDataServer(context.Background(), backingMetadataServer, iamARN, ipv4Address, region, optimistic, container, signer)
 		go notifySystemd()
 		if err := http.Serve(listener, ms); err != nil {
 			return err

config/config.go

@@ -7,6 +7,7 @@ import (
 	"unicode"
 
 	"github.com/Netflix/titus-executor/api/netflix/titus"
+	metadataserverTypes "github.com/Netflix/titus-executor/metadataserver/types"
 	"gopkg.in/urfave/cli.v1"
 )
 
@@ -17,8 +18,12 @@ const (
 // Config contains the executor configuration
 type Config struct {
 	// nolint: maligned
+
 	// MetatronEnabled returns if Metatron is enabled
 	MetatronEnabled bool
+	// Docker image for running the metatron certificate refresh executable
+	ContainerMetatronImage string
+
 	// PrivilegedContainersEnabled returns whether to give tasks CAP_SYS_ADMIN
 	PrivilegedContainersEnabled bool
 	// UseNewNetworkDriver returns which network driver to use
@@ -127,6 +132,13 @@ func NewConfig() (*Config, []cli.Flag) {
 			Destination: &cfg.DockerRegistry,
 			EnvVar:      "DOCKER_REGISTRY",
 		},
+		cli.StringFlag{
+			Name: "container-metatron-image",
+			// This image fetches the task identity document and writes it to `/task-identity`. See `hack/test-images/metatron/`.
+			Value:       "titusoss/metatron@sha256:a850a47bda1238f4bad36fd599679ef518cc40874c0102713982d1058b5a3a88",
+			Destination: &cfg.ContainerMetatronImage,
+			EnvVar:      "CONTAINER_METATRON_IMAGE",
+		},
 		cli.BoolTFlag{
 			Name:        "container-sshd",
 			Destination: &cfg.ContainerSSHD,
@@ -180,12 +192,20 @@ func NewConfig() (*Config, []cli.Flag) {
 }
 
 // GetNetflixEnvForTask fetches the "base" environment configuration, and adds in titus-specific environment variables
-// based on the ContainerInfo, and resources.
+// based on the ContainerInfo, config and resources.
 func (c *Config) GetNetflixEnvForTask(taskInfo *titus.ContainerInfo, mem, cpu, disk, networkBandwidth string) map[string]string {
 	env := c.getEnvHardcoded()
 	env = appendMap(env, c.getEnvFromHost())
 	env = appendMap(env, c.getEnvBasedOnTask(taskInfo, mem, cpu, disk, networkBandwidth))
 	env = appendMap(env, c.getUserProvided(taskInfo))
+
+	if c.MetatronEnabled {
+		// When set, the metadata service will return signed identity documents suitable for bootstrapping Metatron
+		env[metadataserverTypes.TitusMetatronVariableName] = "true"
+	} else {
+		env[metadataserverTypes.TitusMetatronVariableName] = "false"
+	}
+
 	return env
 }