Skip to content
This repository has been archived by the owner on Jan 10, 2023. It is now read-only.

Commit

Permalink
Merge pull request #207 from Netflix/optimistic-metadataserver
Browse files Browse the repository at this point in the history 
Optimistic metadataserver
  • Loading branch information
@sargun
sargun committed Dec 21, 2018
2 parents 4c75f31 + df99885 commit c463ba7
Show file tree
Hide file tree
Showing 7 changed files with 190 additions and 118 deletions.
129 changes: 89 additions & 40 deletions cmd/titus-metadata-service/main.go
View file
Expand Up @@ -2,29 +2,26 @@ package main

import (
"context"
"flag"
"net"
"net/http"
"os"
"strconv"
"syscall"
"time"

"github.com/Netflix/titus-executor/logsutil"
"github.com/Netflix/titus-executor/metadataserver"
"github.com/Netflix/titus-executor/metadataserver/types"
log "github.com/sirupsen/logrus"
"golang.org/x/sys/unix"
"gopkg.in/urfave/cli.v1"
)

// 169 is the first octet of 169.254...
const defaultListeningPort = 8169

var listenerFd int64
var listenPort int
var debug bool
var backingMetadataServer string

/* Either returns a listener, or logs a fatal error */
func getListener() net.Listener {
func getListener(listenPort int, listenerFd int64) net.Listener {
if listenerFd != -1 && listenPort != defaultListeningPort {
log.Fatal("You cannot set both listening port, and listener FD")
}
Expand All @@ -45,7 +42,7 @@ func makeFDListener(fd int64) net.Listener {
if int(r0) == -1 {
log.Fatal("Could not get listener FD because: ", e1)
}
unix.CloseOnExec(int(listenerFd))
unix.CloseOnExec(int(fd))

l, err := net.FileListener(os.NewFile(uintptr(fd), ""))
if err != nil {
Expand All @@ -54,39 +51,91 @@ func makeFDListener(fd int64) net.Listener {
return l
}

func getEnv(key string) string {
val := os.Getenv(key)
if val == "" {
log.WithField("key", key).Fatal("Expected environmental variable unset: ", key)
}
return val

}

func main() {
flag.StringVar(&backingMetadataServer, "backing-metadata-server", "http://169.254.169.254/", "The URI of the AWS metadata server you want to use")
flag.Int64Var(&listenerFd, "listener-fd", -1, "Use a specific fd for listening on")
flag.IntVar(&listenPort, "listener-port", defaultListeningPort, "Use specific port to listen on")
flag.BoolVar(&debug, "debug", false, "Set to true to debug logging")

flag.Parse()
if debug {
log.SetLevel(log.DebugLevel)
} else {
log.SetLevel(log.InfoLevel)
app := cli.NewApp()
app.Name = "titus-metadata-service"
var listenerFd int64
var listenPort int
var debug bool
var backingMetadataServer string
var optimistic bool
var region string
var iamARN string
var titusTaskInstanceID string
var ipv4Address string
app.Flags = []cli.Flag{
cli.StringFlag{
Name: "backing-metadata-server",
Value: "http://169.254.169.254/",
Usage: "The URI of the AWS metadata server you want to use",
Destination: &backingMetadataServer,
},
cli.Int64Flag{
Name: "listener-fd",
Value: -1,
Usage: "Use a specific fd for listening on",
Destination: &listenerFd,
},
cli.BoolFlag{
Name: "debug",
Usage: "Set to true to enable debug logging",
Destination: &debug,
},
cli.IntFlag{
Name: "listener-port",
Value: defaultListeningPort,
Usage: "Use specific port to listen on",
Destination: &listenPort,
},
cli.BoolFlag{
Name: "optimistic",
Usage: "If you set this to to true, the IAM service will optimistically fetch IAM credentials",
Destination: &optimistic,
EnvVar: types.TitusOptimisticIAMVariableName,
},
cli.StringFlag{
Name: "region",
Usage: "The STS service region to use",
Destination: &region,
Value: "",
EnvVar: "EC2_REGION",
},
cli.StringFlag{
Name: "iam-role",
EnvVar: "TITUS_IAM_ROLE",
Destination: &iamARN,
},
cli.StringFlag{
Name: "titus-task-instance-id",
EnvVar: "TITUS_TASK_INSTANCE_ID",
Destination: &titusTaskInstanceID,
},
cli.StringFlag{
Name: "ipv4-address",
EnvVar: "EC2_LOCAL_IPV4",
Destination: &ipv4Address,
},
}
logsutil.MaybeSetupLoggerIfOnJournaldAvailable()

/* Get the requisite configuration from environment variables */
iamARN := getEnv("TITUS_IAM_ROLE")
titusTaskInstanceID := getEnv("TITUS_TASK_INSTANCE_ID")
ipv4Address := getEnv("EC2_LOCAL_IPV4")

listener := getListener()
ms := metadataserver.NewMetaDataServer(context.Background(), backingMetadataServer, iamARN, titusTaskInstanceID, ipv4Address)
go notifySystemd()
if err := http.Serve(listener, ms); err != nil {
log.Fatal(err)
app.Action = func(c *cli.Context) error {
if debug {
log.SetLevel(log.DebugLevel)
} else {
log.SetLevel(log.InfoLevel)
}
logsutil.MaybeSetupLoggerIfOnJournaldAvailable()

/* Get the requisite configuration from environment variables */
listener := getListener(listenPort, listenerFd)
ms := metadataserver.NewMetaDataServer(context.Background(), backingMetadataServer, iamARN, titusTaskInstanceID, ipv4Address, region, optimistic)
go notifySystemd()
if err := http.Serve(listener, ms); err != nil {
return err
}
log.Info("Done serving?")
time.Sleep(1 * time.Second)
return nil
}
if err := app.Run(os.Args); err != nil {
log.WithError(err).Fatal()
}

}
7 changes: 7 additions & 0 deletions executor/runtime/docker/docker.go
View file
Expand Up @@ -25,6 +25,7 @@ import (
"github.com/Netflix/titus-executor/config"
"github.com/Netflix/titus-executor/executor/metatron"
runtimeTypes "github.com/Netflix/titus-executor/executor/runtime/types"
metadataserverTypes "github.com/Netflix/titus-executor/metadataserver/types"
"github.com/Netflix/titus-executor/nvidia"
vpcTypes "github.com/Netflix/titus-executor/vpc/types"
"github.com/docker/docker/api/types"
Expand Down Expand Up @@ -396,6 +397,12 @@ func (r *DockerRuntime) dockerConfig(c *runtimeTypes.Container, binds []string,
return nil, nil, err
}

if optimisticTokenFetch, parseErr := c.GetOptimisticIAMTokenFetch(); parseErr != nil {
return nil, nil, parseErr
} else if optimisticTokenFetch {
c.Env[metadataserverTypes.TitusOptimisticIAMVariableName] = "true"
}

// hostname style: ip-{ip-addr} or {task ID}
hostname, err := c.ComputeHostname()
if err != nil {
Expand Down
16 changes: 13 additions & 3 deletions executor/runtime/types/types.go
View file
Expand Up @@ -26,9 +26,10 @@ import (
const (
hostnameStyleParam = "titusParameter.agent.hostnameStyle"
// FuseEnabledParam is a container atttribute set to enable FUSE
FuseEnabledParam = "titusParameter.agent.fuseEnabled"
assignIPv6AddressParam = "titusParameter.agent.assignIPv6Address"
ttyEnabledParam = "titusParameter.agent.ttyEnabled"
FuseEnabledParam = "titusParameter.agent.fuseEnabled"
assignIPv6AddressParam = "titusParameter.agent.assignIPv6Address"
ttyEnabledParam = "titusParameter.agent.ttyEnabled"
optimisticIAMTokenFetchParam = "titusParameter.agent.optimisticIAMTokenFetch"
)

const (
Expand Down Expand Up @@ -380,6 +381,15 @@ func (c *Container) GetKeepLocalFileAfterUpload() (bool, error) {
return strconv.ParseBool(keepLocalFileAfterUploadStr)
}

// GetOptimisticIAMTokenFetch indicates whether or not we should delete log files after uploading them
func (c *Container) GetOptimisticIAMTokenFetch() (bool, error) {
optimisticIAMTokenFetchStr, ok := c.TitusInfo.GetPassthroughAttributes()[optimisticIAMTokenFetchParam]
if !ok {
return false, nil
}
return strconv.ParseBool(optimisticIAMTokenFetchStr)
}

// Resources specify constraints to be applied to a Container
type Resources struct {
Mem int64 // in MiB
Expand Down

0 comments on commit c463ba7

Please sign in to comment.