tests: add smoke test for app sandbox

stage1/app-rm/app-rm.go

@@ -18,31 +18,41 @@ package main
 
 import (
 	"flag"
+	"fmt"
 	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"syscall"
 
 	"github.com/coreos/rkt/common"
 	rktlog "github.com/coreos/rkt/pkg/log"
 	stage1common "github.com/coreos/rkt/stage1/common"
 	stage1initcommon "github.com/coreos/rkt/stage1/init/common"
 
 	"github.com/appc/spec/schema/types"
+	"github.com/coreos/rkt/pkg/mountinfo"
 )
 
 var (
-	flagApp string
-	debug   bool
-	log     *rktlog.Logger
-	diag    *rktlog.Logger
+	debug     bool
+	flagApp   string
+	flagStage int
+
+	log  *rktlog.Logger
+	diag *rktlog.Logger
 )
 
 func init() {
 	flag.StringVar(&flagApp, "app", "", "Application name")
 	flag.BoolVar(&debug, "debug", false, "Run in debug mode")
+
+	// `--stage` is an internal implementation detail, not part of stage1 contract
+	flag.IntVar(&flagStage, "stage", 0, "Removal step, defaults to 0 when called from the outside")
 }
 
+// This is a multi-step entrypoint. It starts in stage0 context then invokes
+// itself again in stage1 context to perform further cleanup at pod level.
 func main() {
 	flag.Parse()
 
@@ -59,7 +69,31 @@ func main() {
 	}
 
 	enterCmd := stage1common.PrepareEnterCmd(false)
+	switch flagStage {
+	case 0:
+		// clean resources in stage0
+		err = cleanupStage0(appName, enterCmd)
+	case 1:
+		// clean resources in stage1
+		err = cleanupStage1(appName, enterCmd)
+	default:
+		// unknown step
+		err = fmt.Errorf("unsupported cleaning step %d", flagStage)
+	}
+	if err != nil {
+		log.FatalE("cleanup error", err)
+	}
 
+	os.Exit(0)
+}
+
+// cleanupStage0 is the default initial step for rm entrypoint, which takes
+// care of cleaning up resources in stage0 and calling into stage1 by:
+//  1. ensuring that the service has been stopped
+//  2. removing unit files
+//  3. calling itself in stage1 for further cleanups
+//  4. calling `systemctl daemon-reload` in stage1
+func cleanupStage0(appName *types.ACName, enterCmd []string) error {
 	args := enterCmd
 	args = append(args, "/usr/bin/systemctl")
 	args = append(args, "is-active")
@@ -74,7 +108,7 @@ func main() {
 	out, _ := cmd.Output()
 
 	if string(out) != "inactive\n" {
-		log.Fatalf("app %q is still running", appName.String())
+		return fmt.Errorf("app %q is still running", appName.String())
 	}
 
 	s1rootfs := common.Stage1RootfsPath(".")
@@ -86,24 +120,54 @@ func main() {
 
 	for _, p := range appServicePaths {
 		if err := os.Remove(p); err != nil && !os.IsNotExist(err) {
-			log.FatalE("error removing app service file", err)
+			return fmt.Errorf("error removing app service file: %s", err)
 		}
 	}
 
-	args = enterCmd
-	args = append(args, "/usr/bin/systemctl")
-	args = append(args, "daemon-reload")
+	// TODO(sur): find all RW cgroups exposed for this app and clean them up in stage0 context
 
-	cmd = exec.Cmd{
-		Path: args[0],
-		Args: args,
+	// last cleaning steps are performed after entering pod context
+	tasks := [][]string{
+		// inception: call itself to clean stage1 before proceeding
+		{"/app-rm", "--stage=1", fmt.Sprintf("--app=%s", appName), fmt.Sprintf("--debug=%t", debug)},
+		// all cleaned-up, let systemd reload and forget about this app
+		{"/usr/bin/systemctl", "daemon-reload"},
 	}
+	for _, cmdLine := range tasks {
+		args := append(enterCmd, cmdLine...)
+		cmd = exec.Cmd{
+			Path: args[0],
+			Args: args,
+		}
 
-	if out, err := cmd.CombinedOutput(); err != nil {
-		log.Fatalf("%q failed at daemon-reload:\n%s", appName, out)
+		if out, err := cmd.CombinedOutput(); err != nil {
+			return fmt.Errorf("%q removal failed:\n%s", appName, out)
+		}
 	}
+	return nil
+}
 
-	// TODO unmount all the volumes
+// cleanupStage1 is meant to be executed in stage1 context. It inspects pod systemd-pid1 mountinfo to
+// find all remaining mountpoints for appName and proceed to clean them up.
+func cleanupStage1(appName *types.ACName, enterCmd []string) error {
+	// TODO(lucab): re-evaluate once/if we support systemd as non-pid1 (eg. host pid-ns inheriting)
+	mnts, err := mountinfo.ParseMounts(1)
+	if err != nil {
+		return err
+	}
+	appRootFs := filepath.Join("/opt/stage2", appName.String(), "rootfs")
+	mnts = mnts.Filter(mountinfo.HasPrefix(appRootFs))
+
+	// soft-errors here, stage0 may still be able to continue with the removal anyway
+	for _, m := range mnts {
+		// unlink first to avoid back-propagation
+		_ = syscall.Mount("", m.MountPoint, "", syscall.MS_PRIVATE|syscall.MS_REC, "")
+		// simple unmount, it may fail if the target is busy (eg. overlapping children)
+		if e := syscall.Unmount(m.MountPoint, 0); e != nil {
+			// if busy, just detach here and let the kernel clean it once free
+			_ = syscall.Unmount(m.MountPoint, syscall.MNT_DETACH)
+		}
+	}
 
-	os.Exit(0)
+	return nil
 }

tests/rkt_app_sandbox_test.go

@@ -0,0 +1,116 @@
+// Copyright 2016 The rkt Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Disabled on kvm due to https://github.com/coreos/rkt/issues/3382
+// +build !fly,!kvm
+
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/coreos/rkt/tests/testutils"
+)
+
+// TestAppSandboxSmoke is a basic smoke test for `rkt app` sandbox
+// and related commands.
+func TestAppSandboxSmoke(t *testing.T) {
+	actionTimeout := 30 * time.Second
+	imageName := "coreos.com/rkt-inspect/hello"
+	appName := "hello-app"
+	msg := "HelloFromAppInSandbox"
+
+	aciHello := patchTestACI("rkt-inspect-hello.aci", fmt.Sprintf("--name=%s", imageName), fmt.Sprintf("--exec=/inspect --print-msg=%s", msg))
+	defer os.Remove(aciHello)
+
+	ctx := testutils.NewRktRunCtx()
+	defer ctx.Cleanup()
+
+	cmd := strings.Fields(fmt.Sprintf("%s fetch --insecure-options=image %s", ctx.Cmd(), aciHello))
+	fetchCmd := exec.Command(cmd[0], cmd[1:]...)
+	fetchCmd.Env = append(fetchCmd.Env, "RKT_EXPERIMENT_APP=true")
+	fetchOutput, err := fetchCmd.CombinedOutput()
+	if err != nil {
+		t.Fatalf("Unexpected error: %v\n%s", err, fetchOutput)
+	}
+
+	tmpDir := createTempDirOrPanic("rkt-test-cri-")
+	defer os.RemoveAll(tmpDir)
+
+	rktCmd := fmt.Sprintf("%s app sandbox --debug --uuid-file-save=%s/uuid", ctx.Cmd(), tmpDir)
+	err = os.Setenv("RKT_EXPERIMENT_APP", "true")
+	if err != nil {
+		panic(err)
+	}
+	child := spawnOrFail(t, rktCmd)
+	err = os.Unsetenv("RKT_EXPERIMENT_APP")
+	if err != nil {
+		panic(err)
+	}
+
+	expected := "Reached target rkt apps target."
+	if err := expectTimeoutWithOutput(child, expected, actionTimeout); err != nil {
+		t.Fatalf("Expected %q but not found: %v", expected, err)
+	}
+
+	podUUID, err := ioutil.ReadFile(filepath.Join(tmpDir, "uuid"))
+	if err != nil {
+		t.Fatalf("Can't read pod UUID: %v", err)
+	}
+
+	cmd = strings.Fields(fmt.Sprintf("%s app add --debug %s %s --name=%s", ctx.Cmd(), podUUID, imageName, appName))
+	addCmd := exec.Command(cmd[0], cmd[1:]...)
+	addCmd.Env = append(addCmd.Env, "RKT_EXPERIMENT_APP=true")
+	output, err := addCmd.CombinedOutput()
+	if err != nil {
+		t.Fatalf("Unexpected error: %v\n%s", err, output)
+	}
+
+	cmd = strings.Fields(fmt.Sprintf("%s app start --debug %s --app=%s", ctx.Cmd(), podUUID, appName))
+	startCmd := exec.Command(cmd[0], cmd[1:]...)
+	startCmd.Env = append(startCmd.Env, "RKT_EXPERIMENT_APP=true")
+	output, err = startCmd.CombinedOutput()
+	if err != nil {
+		t.Fatalf("Unexpected error: %v\n%s", err, output)
+	}
+
+	if err := expectTimeoutWithOutput(child, msg, actionTimeout); err != nil {
+		t.Fatalf("Expected %q but not found: %v", expected, err)
+	}
+
+	cmd = strings.Fields(fmt.Sprintf("%s app rm --debug %s --app=%s", ctx.Cmd(), podUUID, appName))
+	removeCmd := exec.Command(cmd[0], cmd[1:]...)
+	removeCmd.Env = append(removeCmd.Env, "RKT_EXPERIMENT_APP=true")
+	output, err = removeCmd.CombinedOutput()
+	if err != nil {
+		t.Fatalf("Unexpected error: %v\n%s", err, output)
+	}
+
+	cmd = strings.Fields(fmt.Sprintf("%s stop %s", ctx.Cmd(), podUUID))
+	output, err = exec.Command(cmd[0], cmd[1:]...).CombinedOutput()
+	if err != nil {
+		t.Fatalf("Unexpected error: %v\n%s", err, output)
+	}
+
+	if err := child.Wait(); err != nil {
+		t.Fatalf("rkt didn't terminate correctly: %v", err)
+	}
+}