Join shim into nslistener network ns and end pod support

containerd/api/grpc/server/server.go

@@ -232,6 +232,7 @@ func supervisorContainer2ApiContainer(c *supervisor.Container) *types.Container
 		BundlePath: c.BundlePath,
 		Status:     "running",
 		Runtime:    "runv",
+		Labels:     []string{fmt.Sprintf("nslistener=%d", c.Processes["init"].ProcId)},
 	}
 }
 

create.go

@@ -6,11 +6,14 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"regexp"
+	"strconv"
 	"strings"
 	"syscall"
 	"time"
 
 	"github.com/hyperhq/runv/containerd/api/grpc/types"
+	_ "github.com/hyperhq/runv/nsenter"
 	"github.com/kardianos/osext"
 	"github.com/kr/pty"
 	"github.com/opencontainers/runtime-spec/specs-go"
@@ -97,16 +100,25 @@ func runContainer(context *cli.Context, createOnly bool) error {
 			sharedContainer = spec.Annotations["ocid/sandbox_name"]
 		}
 	} else {
-		for _, ns := range spec.Linux.Namespaces {
+		for i, ns := range spec.Linux.Namespaces {
 			if ns.Path != "" {
-				if strings.Contains(ns.Path, "/") {
-					return fmt.Errorf("Runv doesn't support path to namespace file, it supports containers name as shared namespaces only")
-				}
 				if ns.Type == "mount" {
 					// TODO support it!
 					return fmt.Errorf("Runv doesn't support shared mount namespace currently")
 				}
-				sharedContainer = ns.Path
+				if sharedContainer, err = findSharedContainer(context.GlobalString("root"), ns.Path); err != nil {
+					return fmt.Errorf("failed to find shared container: %v", err)
+				}
+
+				cstate, err := getContainer(context, sharedContainer)
+				if err != nil {
+					return fmt.Errorf("can't get state file for container %q: %v", sharedContainer, err)
+				}
+				spec.Linux.Namespaces[i] = specs.LinuxNamespace{
+					Type: ns.Type,
+					Path: fmt.Sprintf("/proc/%d/ns/%s", cstate.InitProcessPid, ns.Type),
+				}
+
 				_, err = os.Stat(filepath.Join(root, sharedContainer, stateJson))
 				if err != nil {
 					return fmt.Errorf("The container %q is not existing or not ready", sharedContainer)
@@ -115,6 +127,10 @@ func runContainer(context *cli.Context, createOnly bool) error {
 				if err != nil {
 					return fmt.Errorf("The container %q is not ready", sharedContainer)
 				}
+
+				if err = updateSpec(spec, ocffile); err != nil {
+					return fmt.Errorf("update spec file failed: %v", err)
+				}
 			}
 		}
 	}
@@ -238,14 +254,16 @@ func checkConsole(context *cli.Context, p *specs.Process, createOnly bool) error
 // * In runv, shared namespaces multiple containers are located in the same VM which is managed by a runv-daemon.
 // * Any running container can exit in any arbitrary order, the runv-daemon and the VM are existed until the last container of the VM is existed
 
+type createFunction func(stdin, stdout, stderr string) (int, error)
+
 func createContainer(context *cli.Context, container, namespace string, config *specs.Spec) error {
 	address := filepath.Join(namespace, "namespaced.sock")
 	c, err := getClient(address)
 	if err != nil {
 		return err
 	}
 
-	return ociCreate(context, container, "init", func(stdin, stdout, stderr string) error {
+	return ociCreate(context, container, "init", func(stdin, stdout, stderr string) (int, error) {
 		r := &types.CreateContainerRequest{
 			Id:         container,
 			Runtime:    "runv-create",
@@ -255,21 +273,23 @@ func createContainer(context *cli.Context, container, namespace string, config *
 			Stderr:     stderr,
 		}
 
-		if _, err := c.CreateContainer(netcontext.Background(), r); err != nil {
-			return err
+		ctr, err := c.CreateContainer(netcontext.Background(), r)
+		if err != nil {
+			return -1, err
 		}
+		nslPID := retrieveNslPID(ctr.Container.Labels)
 
 		// create symbol link to namespace file
 		namespaceDir := filepath.Join(context.GlobalString("root"), container, "namespace")
 		if err := os.Symlink(namespace, namespaceDir); err != nil {
-			return fmt.Errorf("failed to create symbol link %q: %v", filepath.Join(namespaceDir, "namespaced.sock"), err)
+			return -1, fmt.Errorf("failed to create symbol link %q: %v", filepath.Join(namespaceDir, "namespaced.sock"), err)
 		}
-		return nil
+		return nslPID, nil
 	})
 
 }
 
-func ociCreate(context *cli.Context, container, process string, createFunc func(stdin, stdout, stderr string) error) error {
+func ociCreate(context *cli.Context, container, process string, createFunc createFunction) error {
 	path, err := osext.Executable()
 	if err != nil {
 		return fmt.Errorf("cannot find self executable path for %s: %v\n", os.Args[0], err)
@@ -303,7 +323,7 @@ func ociCreate(context *cli.Context, container, process string, createFunc func(
 		stdout = tty.Name()
 		stderr = tty.Name()
 	}
-	err = createFunc(stdin, stdout, stderr)
+	listenerPID, err := createFunc(stdin, stdout, stderr)
 	if err != nil {
 		return err
 	}
@@ -338,11 +358,15 @@ func ociCreate(context *cli.Context, container, process string, createFunc func(
 				Setsid:  true,
 			},
 		}
+		if listenerPID > 0 {
+			cmd.Env = []string{fmt.Sprintf("_NSLISTENERPID=%d", listenerPID)}
+		}
 		err = cmd.Start()
 		if err != nil {
 			return err
 		}
 	}
+
 	if context.String("pid-file") != "" {
 		err = createPidFile(context.String("pid-file"), cmd.Process.Pid)
 		if err != nil {
@@ -372,3 +396,52 @@ func createPidFile(path string, pid int) error {
 	}
 	return os.Rename(tmpName, path)
 }
+
+func retrieveNslPID(labels []string) int {
+	for _, v := range labels {
+		s := strings.SplitN(v, "=", 2)
+		if len(s) == 2 && s[0] == "nslistener" {
+			pid, err := strconv.Atoi(s[1])
+			if err == nil {
+				return pid
+			}
+		}
+	}
+	return -1
+}
+
+func findSharedContainer(root, nsPath string) (container string, err error) {
+	absRoot, err := filepath.Abs(root)
+	if err != nil {
+		return "", err
+	}
+	list, err := ioutil.ReadDir(absRoot)
+	if err != nil {
+		return "", err
+	}
+
+	if strings.Contains(nsPath, "/") {
+		pidexp := regexp.MustCompile(`/proc/(\d+)/ns/*`)
+		matches := pidexp.FindStringSubmatch(nsPath)
+		if len(matches) != 2 {
+			return "", fmt.Errorf("malformed ns path: %s", nsPath)
+		}
+		pid := matches[1]
+
+		for _, item := range list {
+			shimPidFile := filepath.Join(absRoot, item.Name(), "shim-init.pid")
+			spidByte, err := ioutil.ReadFile(shimPidFile)
+			if err != nil {
+				// for backward compatibility, if dir doesn't contain "shim-init.pid"
+				// it could be old legacy dir, ignore and skip it.
+				continue
+			}
+			spid := strings.TrimSpace(string(spidByte))
+			if pid == spid {
+				return item.Name(), nil
+			}
+		}
+		return "", fmt.Errorf("can't find container with shim pid %s", pid)
+	}
+	return nsPath, nil
+}

exec.go

@@ -208,7 +208,7 @@ func runProcess(context *cli.Context, container string, config *specs.Process) (
 		monitorTtySize(c, container, process)
 	}
 
-	err = ociCreate(context, container, process, func(stdin, stdout, stderr string) error {
+	err = ociCreate(context, container, process, func(stdin, stdout, stderr string) (int, error) {
 		p := &types.AddProcessRequest{
 			Id:       container,
 			Pid:      process,
@@ -225,9 +225,9 @@ func runProcess(context *cli.Context, container string, config *specs.Process) (
 			Stderr: stderr,
 		}
 		if _, err := c.AddProcess(netcontext.Background(), p); err != nil {
-			return err
+			return -1, err
 		}
-		return nil
+		return -1, nil
 	})
 	if err != nil {
 		return -1, err

list.go

@@ -50,7 +50,7 @@ in json format:
 		},
 	},
 	Action: func(context *cli.Context) {
-		s, err := getContainers(context)
+		s, err := getContainers(context.GlobalString("root"))
 		if err != nil {
 			fatal(err)
 		}
@@ -91,8 +91,7 @@ in json format:
 	},
 }
 
-func getContainers(context *cli.Context) ([]containerState, error) {
-	root := context.GlobalString("root")
+func getContainers(root string) ([]containerState, error) {
 	absRoot, err := filepath.Abs(root)
 	if err != nil {
 		return nil, err

nsenter/namespace.h

@@ -0,0 +1,14 @@
+#ifndef NSENTER_NAMESPACE_H
+#define NSENTER_NAMESPACE_H
+
+#ifndef _GNU_SOURCE
+#	define _GNU_SOURCE
+#endif
+#include <sched.h>
+
+/* All of these are taken from include/uapi/linux/sched.h */
+#ifndef CLONE_NEWNET
+#	define CLONE_NEWNET 0x40000000 /* New network namespace */
+#endif
+
+#endif /* NSENTER_NAMESPACE_H */

nsenter/nsenter.go

@@ -0,0 +1,12 @@
+// +build linux,!gccgo
+
+package nsenter
+
+/*
+#cgo CFLAGS: -Wall
+extern void nsexec();
+void __attribute__((constructor)) init(void) {
+	nsexec();
+}
+*/
+import "C"

nsenter/nsenter_gccgo.go

@@ -0,0 +1,25 @@
+// +build linux,gccgo
+
+package nsenter
+
+/*
+#cgo CFLAGS: -Wall
+extern void nsexec();
+void __attribute__((constructor)) init(void) {
+	nsexec();
+}
+*/
+import "C"
+
+// AlwaysFalse is here to stay false
+// (and be exported so the compiler doesn't optimize out its reference)
+var AlwaysFalse bool
+
+func init() {
+	if AlwaysFalse {
+		// by referencing this C init() in a noop test, it will ensure the compiler
+		// links in the C function.
+		// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=65134
+		C.init()
+	}
+}

nsenter/nsenter_unsupported.go

@@ -0,0 +1,5 @@
+// +build !linux !cgo
+
+package nsenter
+
+import "C"

nsenter/nsexec.c

@@ -0,0 +1,54 @@
+#define _GNU_SOURCE
+#include <stdio.h>
+#include <stdlib.h>
+#include <sched.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+
+/* Get all of the CLONE_NEW* flags. */
+#include "namespace.h"
+
+#define bail(fmt, ...)								\
+	do {									\
+		int ret = __COUNTER__ + 1;					\
+		fprintf(stderr, "nsenter: " fmt ": %m\n", ##__VA_ARGS__);	\
+		exit(ret);							\
+	} while(0)
+
+static int getNslistenerPid(void)
+{
+	int pid;
+	char *nspidStr, *endptr;
+
+	nspidStr = getenv("_NSLISTENERPID");
+	if (nspidStr== NULL || *nspidStr == '\0')
+		return -1;
+
+	pid = strtol(nspidStr, &endptr, 10);
+	if (*endptr != '\0')
+		bail("unable to parse _NSLISTENERPID");
+
+	return pid;
+}
+
+void nsexec(void)
+{
+	int nsPid, nsFd;
+	char *path;
+
+	nsPid = getNslistenerPid();
+	if (nsPid <= 0) {
+		return;
+	}
+
+	path = malloc(sizeof(char)*64);
+	sprintf(path, "/proc/%d/ns/net", nsPid);
+	nsFd = open(path, O_RDONLY);
+	setns(nsFd, CLONE_NEWNET);
+
+	free(path);
+	close(nsFd);
+	return;
+}

shim.go

@@ -65,6 +65,26 @@ var shimCommand = cli.Command{
 			defer signal.Stop(sigc)
 		}
 
+		stateDir := filepath.Join(root, container)
+		if _, err = os.Stat(stateDir); err == nil {
+			// state dir exist, write shim pid into it
+			shimFile := filepath.Join(stateDir, "shim-"+process+".pid")
+			f, err := os.OpenFile(shimFile, os.O_WRONLY|os.O_CREATE, 0640)
+			if err != nil {
+				glog.Errorf("can't create shim pid file: %v", err)
+				goto eventHandle
+			}
+			defer f.Close()
+			shimPid := fmt.Sprintf("%d", os.Getpid())
+			_, err = f.Write([]byte(shimPid))
+			if err != nil {
+				glog.Errorf("can't write pid %q to shim pid file: %v", shimPid, err)
+				goto eventHandle
+			}
+			defer os.Remove(shimFile)
+		}
+
+	eventHandle:
 		// wait until exit
 		evChan := containerEvents(c, container)
 		for e := range evChan {

spec.go

@@ -103,3 +103,23 @@ func loadSpec(ocffile string) (*specs.Spec, error) {
 	}
 	return &spec, nil
 }
+
+func updateSpec(spec *specs.Spec, ocffile string) error {
+	if _, err := os.Stat(ocffile); err != nil {
+		if os.IsNotExist(err) {
+			return fmt.Errorf("%q doesn't exists", ocffile)
+		}
+		return fmt.Errorf("Stat %q error: %v", ocffile, err)
+	}
+
+	data, err := json.Marshal(spec)
+	if err != nil {
+		return fmt.Errorf("failed to marshal spec file during update: %v", err)
+	}
+
+	if err = ioutil.WriteFile(ocffile, data, 0640); err != nil {
+		return err
+	}
+
+	return nil
+}