Skip to content
This repository has been archived by the owner on Jan 10, 2023. It is now read-only.

Optimistic metadataserver #207

Merged
merged 1 commit into from
Dec 21, 2018
Merged

Optimistic metadataserver #207

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
129 changes: 89 additions & 40 deletions cmd/titus-metadata-service/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,29 +2,26 @@ package main

import (
"context"
"flag"
"net"
"net/http"
"os"
"strconv"
"syscall"
"time"

"github.com/Netflix/titus-executor/logsutil"
"github.com/Netflix/titus-executor/metadataserver"
"github.com/Netflix/titus-executor/metadataserver/types"
log "github.com/sirupsen/logrus"
"golang.org/x/sys/unix"
"gopkg.in/urfave/cli.v1"
)

// 169 is the first octet of 169.254...
const defaultListeningPort = 8169

var listenerFd int64
var listenPort int
var debug bool
var backingMetadataServer string

/* Either returns a listener, or logs a fatal error */
func getListener() net.Listener {
func getListener(listenPort int, listenerFd int64) net.Listener {
if listenerFd != -1 && listenPort != defaultListeningPort {
log.Fatal("You cannot set both listening port, and listener FD")
}
Expand All @@ -45,7 +42,7 @@ func makeFDListener(fd int64) net.Listener {
if int(r0) == -1 {
log.Fatal("Could not get listener FD because: ", e1)
}
unix.CloseOnExec(int(listenerFd))
unix.CloseOnExec(int(fd))

l, err := net.FileListener(os.NewFile(uintptr(fd), ""))
if err != nil {
Expand All @@ -54,39 +51,91 @@ func makeFDListener(fd int64) net.Listener {
return l
}

func getEnv(key string) string {
val := os.Getenv(key)
if val == "" {
log.WithField("key", key).Fatal("Expected environmental variable unset: ", key)
}
return val

}

func main() {
flag.StringVar(&backingMetadataServer, "backing-metadata-server", "http://169.254.169.254/", "The URI of the AWS metadata server you want to use")
flag.Int64Var(&listenerFd, "listener-fd", -1, "Use a specific fd for listening on")
flag.IntVar(&listenPort, "listener-port", defaultListeningPort, "Use specific port to listen on")
flag.BoolVar(&debug, "debug", false, "Set to true to debug logging")

flag.Parse()
if debug {
log.SetLevel(log.DebugLevel)
} else {
log.SetLevel(log.InfoLevel)
app := cli.NewApp()
app.Name = "titus-metadata-service"
var listenerFd int64
var listenPort int
var debug bool
var backingMetadataServer string
var optimistic bool
var region string
var iamARN string
var titusTaskInstanceID string
var ipv4Address string
app.Flags = []cli.Flag{
cli.StringFlag{
Name: "backing-metadata-server",
Value: "http://169.254.169.254/",
Usage: "The URI of the AWS metadata server you want to use",
Destination: &backingMetadataServer,
},
cli.Int64Flag{
Name: "listener-fd",
Value: -1,
Usage: "Use a specific fd for listening on",
Destination: &listenerFd,
},
cli.BoolFlag{
Name: "debug",
Usage: "Set to true to enable debug logging",
Destination: &debug,
},
cli.IntFlag{
Name: "listener-port",
Value: defaultListeningPort,
Usage: "Use specific port to listen on",
Destination: &listenPort,
},
cli.BoolFlag{
Name: "optimistic",
Usage: "If you set this to to true, the IAM service will optimistically fetch IAM credentials",
Destination: &optimistic,
EnvVar: types.TitusOptimisticIAMVariableName,
},
cli.StringFlag{
Name: "region",
Usage: "The STS service region to use",
Destination: &region,
Value: "",
EnvVar: "EC2_REGION",
},
cli.StringFlag{
Name: "iam-role",
EnvVar: "TITUS_IAM_ROLE",
Destination: &iamARN,
},
cli.StringFlag{
Name: "titus-task-instance-id",
EnvVar: "TITUS_TASK_INSTANCE_ID",
Destination: &titusTaskInstanceID,
},
cli.StringFlag{
Name: "ipv4-address",
EnvVar: "EC2_LOCAL_IPV4",
Destination: &ipv4Address,
},
}
logsutil.MaybeSetupLoggerIfOnJournaldAvailable()

/* Get the requisite configuration from environment variables */
iamARN := getEnv("TITUS_IAM_ROLE")
titusTaskInstanceID := getEnv("TITUS_TASK_INSTANCE_ID")
ipv4Address := getEnv("EC2_LOCAL_IPV4")

listener := getListener()
ms := metadataserver.NewMetaDataServer(context.Background(), backingMetadataServer, iamARN, titusTaskInstanceID, ipv4Address)
go notifySystemd()
if err := http.Serve(listener, ms); err != nil {
log.Fatal(err)
app.Action = func(c *cli.Context) error {
if debug {
log.SetLevel(log.DebugLevel)
} else {
log.SetLevel(log.InfoLevel)
}
logsutil.MaybeSetupLoggerIfOnJournaldAvailable()

/* Get the requisite configuration from environment variables */
listener := getListener(listenPort, listenerFd)
ms := metadataserver.NewMetaDataServer(context.Background(), backingMetadataServer, iamARN, titusTaskInstanceID, ipv4Address, region, optimistic)
go notifySystemd()
if err := http.Serve(listener, ms); err != nil {
return err
}
log.Info("Done serving?")
time.Sleep(1 * time.Second)
return nil
}
if err := app.Run(os.Args); err != nil {
log.WithError(err).Fatal()
}

}
7 changes: 7 additions & 0 deletions executor/runtime/docker/docker.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
"github.com/Netflix/titus-executor/config"
"github.com/Netflix/titus-executor/executor/metatron"
runtimeTypes "github.com/Netflix/titus-executor/executor/runtime/types"
metadataserverTypes "github.com/Netflix/titus-executor/metadataserver/types"
"github.com/Netflix/titus-executor/nvidia"
vpcTypes "github.com/Netflix/titus-executor/vpc/types"
"github.com/docker/docker/api/types"
Expand Down Expand Up @@ -396,6 +397,12 @@ func (r *DockerRuntime) dockerConfig(c *runtimeTypes.Container, binds []string,
return nil, nil, err
}

if optimisticTokenFetch, parseErr := c.GetOptimisticIAMTokenFetch(); parseErr != nil {
return nil, nil, parseErr
} else if optimisticTokenFetch {
c.Env[metadataserverTypes.TitusOptimisticIAMVariableName] = "true"
}

// hostname style: ip-{ip-addr} or {task ID}
hostname, err := c.ComputeHostname()
if err != nil {
Expand Down
16 changes: 13 additions & 3 deletions executor/runtime/types/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,10 @@ import (
const (
hostnameStyleParam = "titusParameter.agent.hostnameStyle"
// FuseEnabledParam is a container atttribute set to enable FUSE
FuseEnabledParam = "titusParameter.agent.fuseEnabled"
assignIPv6AddressParam = "titusParameter.agent.assignIPv6Address"
ttyEnabledParam = "titusParameter.agent.ttyEnabled"
FuseEnabledParam = "titusParameter.agent.fuseEnabled"
assignIPv6AddressParam = "titusParameter.agent.assignIPv6Address"
ttyEnabledParam = "titusParameter.agent.ttyEnabled"
optimisticIAMTokenFetchParam = "titusParameter.agent.optimisticIAMTokenFetch"
)

const (
Expand Down Expand Up @@ -380,6 +381,15 @@ func (c *Container) GetKeepLocalFileAfterUpload() (bool, error) {
return strconv.ParseBool(keepLocalFileAfterUploadStr)
}

// GetOptimisticIAMTokenFetch indicates whether or not we should delete log files after uploading them
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Copy and pasta?

func (c *Container) GetOptimisticIAMTokenFetch() (bool, error) {
optimisticIAMTokenFetchStr, ok := c.TitusInfo.GetPassthroughAttributes()[optimisticIAMTokenFetchParam]
if !ok {
return false, nil
}
return strconv.ParseBool(optimisticIAMTokenFetchStr)
}

// Resources specify constraints to be applied to a Container
type Resources struct {
Mem int64 // in MiB
Expand Down
Loading