Merge pull request #165 from Netflix/fuse

Lock Down FUSE

executor/runtime/docker/capabilities.go

@@ -0,0 +1,72 @@
+package docker
+
+import (
+	"github.com/Netflix/titus-executor/executor/runtime/docker/seccomp"
+	runtimeTypes "github.com/Netflix/titus-executor/executor/runtime/types"
+	"github.com/docker/docker/api/types/container"
+
+	"fmt"
+)
+
+const (
+	SYS_ADMIN = "SYS_ADMIN" // nolint: golint
+)
+
+func addAdditionalCapabilities(c *runtimeTypes.Container, hostCfg *container.HostConfig) map[string]struct{} {
+	addedCapabilities := make(map[string]struct{})
+
+	// Set any additional capabilities for this container
+	if cap := c.TitusInfo.GetCapabilities(); cap != nil {
+		for _, add := range cap.GetAdd() {
+			addedCapabilities[add.String()] = struct{}{}
+			hostCfg.CapAdd = append(hostCfg.CapAdd, add.String())
+		}
+		for _, drop := range cap.GetDrop() {
+			hostCfg.CapDrop = append(hostCfg.CapDrop, drop.String())
+		}
+	}
+	return addedCapabilities
+}
+
+func setupAdditionalCapabilities(c *runtimeTypes.Container, hostCfg *container.HostConfig) error {
+	addedCapabilities := addAdditionalCapabilities(c, hostCfg)
+
+	// Privileged containers automaticaly deactivate seccomp and friends, no need to do this
+	fuseEnabled, err := c.GetFuseEnabled()
+	if err != nil {
+		return err
+	}
+	if fuseEnabled || c.TitusInfo.GetAllowNestedContainers() {
+		if _, ok := addedCapabilities[SYS_ADMIN]; !ok {
+			hostCfg.CapAdd = append(hostCfg.CapAdd, SYS_ADMIN)
+		}
+	}
+	seccompProfile := "default.json"
+	apparmorProfile := ""
+
+	if fuseEnabled {
+		hostCfg.Resources.Devices = append(hostCfg.Resources.Devices, container.DeviceMapping{
+			PathOnHost:        fuseDev,
+			PathInContainer:   fuseDev,
+			CgroupPermissions: "rmw",
+		})
+		apparmorProfile = "docker-fuse"
+		seccompProfile = "fuse-container.json"
+
+	}
+	// We can do this here because nested containers can do everything fuse containers can
+	if c.TitusInfo.GetAllowNestedContainers() {
+		apparmorProfile = "docker-nested"
+		seccompProfile = "nested-container.json"
+
+		c.Env["TINI_HANDOFF"] = trueString
+		c.Env["TINI_UNSHARE"] = trueString
+	}
+
+	if apparmorProfile != "" {
+		hostCfg.SecurityOpt = append(hostCfg.SecurityOpt, "apparmor:"+apparmorProfile)
+	}
+	hostCfg.SecurityOpt = append(hostCfg.SecurityOpt, fmt.Sprintf("seccomp=%s", string(seccomp.MustAsset(seccompProfile))))
+
+	return nil
+}

executor/runtime/docker/capabilities_test.go

@@ -0,0 +1,80 @@
+package docker
+
+import (
+	"testing"
+
+	"github.com/Netflix/titus-executor/api/netflix/titus"
+	runtimeTypes "github.com/Netflix/titus-executor/executor/runtime/types"
+	"github.com/docker/docker/api/types/container"
+	"github.com/gogo/protobuf/proto"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDefaultProfile(t *testing.T) {
+	c := runtimeTypes.Container{
+		TitusInfo: &titus.ContainerInfo{},
+	}
+	hostConfig := container.HostConfig{}
+
+	assert.NoError(t, setupAdditionalCapabilities(&c, &hostConfig))
+
+	assert.Len(t, hostConfig.CapAdd, 0)
+	assert.Len(t, hostConfig.CapDrop, 0)
+	assert.Len(t, hostConfig.SecurityOpt, 1)
+}
+
+func TestFuseProfile(t *testing.T) {
+	c := runtimeTypes.Container{
+		TitusInfo: &titus.ContainerInfo{
+			PassthroughAttributes: map[string]string{
+				runtimeTypes.FuseEnabledParam: "true",
+			},
+		},
+	}
+	hostConfig := container.HostConfig{}
+
+	assert.NoError(t, setupAdditionalCapabilities(&c, &hostConfig))
+
+	assert.Contains(t, hostConfig.CapAdd, "SYS_ADMIN")
+	assert.Len(t, hostConfig.CapDrop, 0)
+	assert.Len(t, hostConfig.SecurityOpt, 2)
+	assert.Contains(t, hostConfig.SecurityOpt, "apparmor:docker-fuse")
+}
+
+func TestNestedContainerProfile(t *testing.T) {
+	c := runtimeTypes.Container{
+		Env: map[string]string{},
+		TitusInfo: &titus.ContainerInfo{
+			AllowNestedContainers: proto.Bool(true),
+		},
+	}
+	hostConfig := container.HostConfig{}
+
+	assert.NoError(t, setupAdditionalCapabilities(&c, &hostConfig))
+
+	assert.Contains(t, hostConfig.CapAdd, "SYS_ADMIN")
+	assert.Len(t, hostConfig.CapDrop, 0)
+	assert.Len(t, hostConfig.SecurityOpt, 2)
+	assert.Contains(t, hostConfig.SecurityOpt, "apparmor:docker-nested")
+
+}
+
+func TestFuseAndNestedContainerProfileProfile(t *testing.T) {
+	c := runtimeTypes.Container{
+		Env: map[string]string{},
+		TitusInfo: &titus.ContainerInfo{
+			AllowNestedContainers: proto.Bool(true),
+			PassthroughAttributes: map[string]string{
+				runtimeTypes.FuseEnabledParam: "true",
+			},
+		},
+	}
+	hostConfig := container.HostConfig{}
+
+	assert.NoError(t, setupAdditionalCapabilities(&c, &hostConfig))
+
+	assert.Contains(t, hostConfig.CapAdd, "SYS_ADMIN")
+	assert.Len(t, hostConfig.CapDrop, 0)
+	assert.Len(t, hostConfig.SecurityOpt, 2)
+	assert.Contains(t, hostConfig.SecurityOpt, "apparmor:docker-nested")
+}

executor/runtime/docker/docker.go

@@ -24,7 +24,6 @@ import (
 	"github.com/Netflix/titus-executor/api/netflix/titus"
 	"github.com/Netflix/titus-executor/config"
 	"github.com/Netflix/titus-executor/executor/metatron"
-	"github.com/Netflix/titus-executor/executor/runtime/docker/seccomp"
 	runtimeTypes "github.com/Netflix/titus-executor/executor/runtime/types"
 	"github.com/Netflix/titus-executor/nvidia"
 	vpcTypes "github.com/Netflix/titus-executor/vpc/types"
@@ -480,7 +479,10 @@ func (r *DockerRuntime) dockerConfig(c *runtimeTypes.Container, binds []string,
 		// seccomp profile will automatically adjust based on the capabilities.
 		hostCfg.SecurityOpt = append(hostCfg.SecurityOpt, "apparmor:unconfined")
 	} else {
-		r.setupAdditionalCapabilities(c, hostCfg)
+		err = setupAdditionalCapabilities(c, hostCfg)
+		if err != nil {
+			return nil, nil, err
+		}
 	}
 
 	// label is necessary for metadata proxy compatibility
@@ -539,37 +541,6 @@ func netflixLoggerTempDir(cfg config.Config, c *runtimeTypes.Container) string {
 	return filepath.Join(cfg.LogsTmpDir, c.TaskID)
 }
 
-func (r *DockerRuntime) setupAdditionalCapabilities(c *runtimeTypes.Container, hostCfg *container.HostConfig) {
-	addedCapabilities := make(map[string]struct{})
-
-	// Set any additional capabilities for this container
-	if cap := c.TitusInfo.GetCapabilities(); cap != nil {
-		for _, add := range cap.GetAdd() {
-			addedCapabilities[add.String()] = struct{}{}
-			hostCfg.CapAdd = append(hostCfg.CapAdd, add.String())
-		}
-		for _, drop := range cap.GetDrop() {
-			hostCfg.CapDrop = append(hostCfg.CapDrop, drop.String())
-		}
-	}
-
-	// Privileged containers automaticaly deactivate seccomp and friends, no need to do this
-	if c.TitusInfo.GetAllowNestedContainers() {
-		hostCfg.SecurityOpt = append(hostCfg.SecurityOpt, "apparmor:docker-nested")
-
-		hostCfg.SecurityOpt = append(hostCfg.SecurityOpt, fmt.Sprintf("seccomp=%s", string(seccomp.MustAsset("nested-container.json"))))
-
-		if _, ok := addedCapabilities["SYS_ADMIN"]; !ok {
-			hostCfg.CapAdd = append(hostCfg.CapAdd, "SYS_ADMIN")
-		}
-		c.Env["TINI_HANDOFF"] = trueString
-		c.Env["TINI_UNSHARE"] = trueString
-	} else {
-		hostCfg.SecurityOpt = append(hostCfg.SecurityOpt, fmt.Sprintf("seccomp=%s", string(seccomp.MustAsset("default.json"))))
-	}
-
-}
-
 func sleepWithCtx(parentCtx context.Context, d time.Duration) error {
 	select {
 	case <-parentCtx.Done():

executor/runtime/docker/seccomp/Makefile

@@ -2,6 +2,6 @@
 
 all: seccomp.go
 
-seccomp.go: default.json nested-container.json
-	go-bindata -pkg seccomp -o seccomp.go default.json nested-container.json
+seccomp.go: default.json nested-container.json fuse-container.json
+	go-bindata -pkg seccomp -o seccomp.go default.json nested-container.json fuse-container.json