diff --git a/containerd/api/grpc/server/server.go b/containerd/api/grpc/server/server.go index fbf5175e..c24ee32e 100644 --- a/containerd/api/grpc/server/server.go +++ b/containerd/api/grpc/server/server.go @@ -232,6 +232,7 @@ func supervisorContainer2ApiContainer(c *supervisor.Container) *types.Container BundlePath: c.BundlePath, Status: "running", Runtime: "runv", + Labels: []string{fmt.Sprintf("nslistener=%d", c.Processes["init"].ProcId)}, } } diff --git a/create.go b/create.go index d7224322..0f63c440 100644 --- a/create.go +++ b/create.go @@ -6,11 +6,14 @@ import ( "os" "os/exec" "path/filepath" + "regexp" + "strconv" "strings" "syscall" "time" "github.com/hyperhq/runv/containerd/api/grpc/types" + _ "github.com/hyperhq/runv/nsenter" "github.com/kardianos/osext" "github.com/kr/pty" "github.com/opencontainers/runtime-spec/specs-go" @@ -97,16 +100,25 @@ func runContainer(context *cli.Context, createOnly bool) error { sharedContainer = spec.Annotations["ocid/sandbox_name"] } } else { - for _, ns := range spec.Linux.Namespaces { + for i, ns := range spec.Linux.Namespaces { if ns.Path != "" { - if strings.Contains(ns.Path, "/") { - return fmt.Errorf("Runv doesn't support path to namespace file, it supports containers name as shared namespaces only") - } if ns.Type == "mount" { // TODO support it! return fmt.Errorf("Runv doesn't support shared mount namespace currently") } - sharedContainer = ns.Path + if sharedContainer, err = findSharedContainer(context.GlobalString("root"), ns.Path); err != nil { + return fmt.Errorf("failed to find shared container: %v", err) + } + + cstate, err := getContainer(context, sharedContainer) + if err != nil { + return fmt.Errorf("can't get state file for container %q: %v", sharedContainer, err) + } + spec.Linux.Namespaces[i] = specs.LinuxNamespace{ + Type: ns.Type, + Path: fmt.Sprintf("/proc/%d/ns/%s", cstate.InitProcessPid, ns.Type), + } + _, err = os.Stat(filepath.Join(root, sharedContainer, stateJson)) if err != nil { return fmt.Errorf("The container %q is not existing or not ready", sharedContainer) @@ -115,6 +127,10 @@ func runContainer(context *cli.Context, createOnly bool) error { if err != nil { return fmt.Errorf("The container %q is not ready", sharedContainer) } + + if err = updateSpec(spec, ocffile); err != nil { + return fmt.Errorf("update spec file failed: %v", err) + } } } } @@ -238,6 +254,8 @@ func checkConsole(context *cli.Context, p *specs.Process, createOnly bool) error // * In runv, shared namespaces multiple containers are located in the same VM which is managed by a runv-daemon. // * Any running container can exit in any arbitrary order, the runv-daemon and the VM are existed until the last container of the VM is existed +type createFunction func(stdin, stdout, stderr string) (int, error) + func createContainer(context *cli.Context, container, namespace string, config *specs.Spec) error { address := filepath.Join(namespace, "namespaced.sock") c, err := getClient(address) @@ -245,7 +263,7 @@ func createContainer(context *cli.Context, container, namespace string, config * return err } - return ociCreate(context, container, "init", func(stdin, stdout, stderr string) error { + return ociCreate(context, container, "init", func(stdin, stdout, stderr string) (int, error) { r := &types.CreateContainerRequest{ Id: container, Runtime: "runv-create", @@ -255,21 +273,23 @@ func createContainer(context *cli.Context, container, namespace string, config * Stderr: stderr, } - if _, err := c.CreateContainer(netcontext.Background(), r); err != nil { - return err + ctr, err := c.CreateContainer(netcontext.Background(), r) + if err != nil { + return -1, err } + nslPID := retrieveNslPID(ctr.Container.Labels) // create symbol link to namespace file namespaceDir := filepath.Join(context.GlobalString("root"), container, "namespace") if err := os.Symlink(namespace, namespaceDir); err != nil { - return fmt.Errorf("failed to create symbol link %q: %v", filepath.Join(namespaceDir, "namespaced.sock"), err) + return -1, fmt.Errorf("failed to create symbol link %q: %v", filepath.Join(namespaceDir, "namespaced.sock"), err) } - return nil + return nslPID, nil }) } -func ociCreate(context *cli.Context, container, process string, createFunc func(stdin, stdout, stderr string) error) error { +func ociCreate(context *cli.Context, container, process string, createFunc createFunction) error { path, err := osext.Executable() if err != nil { return fmt.Errorf("cannot find self executable path for %s: %v\n", os.Args[0], err) @@ -303,7 +323,7 @@ func ociCreate(context *cli.Context, container, process string, createFunc func( stdout = tty.Name() stderr = tty.Name() } - err = createFunc(stdin, stdout, stderr) + listenerPID, err := createFunc(stdin, stdout, stderr) if err != nil { return err } @@ -338,11 +358,15 @@ func ociCreate(context *cli.Context, container, process string, createFunc func( Setsid: true, }, } + if listenerPID > 0 { + cmd.Env = []string{fmt.Sprintf("_NSLISTENERPID=%d", listenerPID)} + } err = cmd.Start() if err != nil { return err } } + if context.String("pid-file") != "" { err = createPidFile(context.String("pid-file"), cmd.Process.Pid) if err != nil { @@ -372,3 +396,52 @@ func createPidFile(path string, pid int) error { } return os.Rename(tmpName, path) } + +func retrieveNslPID(labels []string) int { + for _, v := range labels { + s := strings.SplitN(v, "=", 2) + if len(s) == 2 && s[0] == "nslistener" { + pid, err := strconv.Atoi(s[1]) + if err == nil { + return pid + } + } + } + return -1 +} + +func findSharedContainer(root, nsPath string) (container string, err error) { + absRoot, err := filepath.Abs(root) + if err != nil { + return "", err + } + list, err := ioutil.ReadDir(absRoot) + if err != nil { + return "", err + } + + if strings.Contains(nsPath, "/") { + pidexp := regexp.MustCompile(`/proc/(\d+)/ns/*`) + matches := pidexp.FindStringSubmatch(nsPath) + if len(matches) != 2 { + return "", fmt.Errorf("malformed ns path: %s", nsPath) + } + pid := matches[1] + + for _, item := range list { + shimPidFile := filepath.Join(absRoot, item.Name(), "shim-init.pid") + spidByte, err := ioutil.ReadFile(shimPidFile) + if err != nil { + // for backward compatibility, if dir doesn't contain "shim-init.pid" + // it could be old legacy dir, ignore and skip it. + continue + } + spid := strings.TrimSpace(string(spidByte)) + if pid == spid { + return item.Name(), nil + } + } + return "", fmt.Errorf("can't find container with shim pid %s", pid) + } + return nsPath, nil +} diff --git a/exec.go b/exec.go index a4f3ab48..29a02e1b 100644 --- a/exec.go +++ b/exec.go @@ -208,7 +208,7 @@ func runProcess(context *cli.Context, container string, config *specs.Process) ( monitorTtySize(c, container, process) } - err = ociCreate(context, container, process, func(stdin, stdout, stderr string) error { + err = ociCreate(context, container, process, func(stdin, stdout, stderr string) (int, error) { p := &types.AddProcessRequest{ Id: container, Pid: process, @@ -225,9 +225,9 @@ func runProcess(context *cli.Context, container string, config *specs.Process) ( Stderr: stderr, } if _, err := c.AddProcess(netcontext.Background(), p); err != nil { - return err + return -1, err } - return nil + return -1, nil }) if err != nil { return -1, err diff --git a/list.go b/list.go index 5aabc4a4..24c82387 100644 --- a/list.go +++ b/list.go @@ -50,7 +50,7 @@ in json format: }, }, Action: func(context *cli.Context) { - s, err := getContainers(context) + s, err := getContainers(context.GlobalString("root")) if err != nil { fatal(err) } @@ -91,8 +91,7 @@ in json format: }, } -func getContainers(context *cli.Context) ([]containerState, error) { - root := context.GlobalString("root") +func getContainers(root string) ([]containerState, error) { absRoot, err := filepath.Abs(root) if err != nil { return nil, err diff --git a/nsenter/namespace.h b/nsenter/namespace.h new file mode 100644 index 00000000..d62ffd36 --- /dev/null +++ b/nsenter/namespace.h @@ -0,0 +1,14 @@ +#ifndef NSENTER_NAMESPACE_H +#define NSENTER_NAMESPACE_H + +#ifndef _GNU_SOURCE +# define _GNU_SOURCE +#endif +#include + +/* All of these are taken from include/uapi/linux/sched.h */ +#ifndef CLONE_NEWNET +# define CLONE_NEWNET 0x40000000 /* New network namespace */ +#endif + +#endif /* NSENTER_NAMESPACE_H */ diff --git a/nsenter/nsenter.go b/nsenter/nsenter.go new file mode 100644 index 00000000..07f4d63e --- /dev/null +++ b/nsenter/nsenter.go @@ -0,0 +1,12 @@ +// +build linux,!gccgo + +package nsenter + +/* +#cgo CFLAGS: -Wall +extern void nsexec(); +void __attribute__((constructor)) init(void) { + nsexec(); +} +*/ +import "C" diff --git a/nsenter/nsenter_gccgo.go b/nsenter/nsenter_gccgo.go new file mode 100644 index 00000000..63c7a3ec --- /dev/null +++ b/nsenter/nsenter_gccgo.go @@ -0,0 +1,25 @@ +// +build linux,gccgo + +package nsenter + +/* +#cgo CFLAGS: -Wall +extern void nsexec(); +void __attribute__((constructor)) init(void) { + nsexec(); +} +*/ +import "C" + +// AlwaysFalse is here to stay false +// (and be exported so the compiler doesn't optimize out its reference) +var AlwaysFalse bool + +func init() { + if AlwaysFalse { + // by referencing this C init() in a noop test, it will ensure the compiler + // links in the C function. + // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=65134 + C.init() + } +} diff --git a/nsenter/nsenter_unsupported.go b/nsenter/nsenter_unsupported.go new file mode 100644 index 00000000..ac701ca3 --- /dev/null +++ b/nsenter/nsenter_unsupported.go @@ -0,0 +1,5 @@ +// +build !linux !cgo + +package nsenter + +import "C" diff --git a/nsenter/nsexec.c b/nsenter/nsexec.c new file mode 100644 index 00000000..f6b3464b --- /dev/null +++ b/nsenter/nsexec.c @@ -0,0 +1,54 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include + +/* Get all of the CLONE_NEW* flags. */ +#include "namespace.h" + +#define bail(fmt, ...) \ + do { \ + int ret = __COUNTER__ + 1; \ + fprintf(stderr, "nsenter: " fmt ": %m\n", ##__VA_ARGS__); \ + exit(ret); \ + } while(0) + +static int getNslistenerPid(void) +{ + int pid; + char *nspidStr, *endptr; + + nspidStr = getenv("_NSLISTENERPID"); + if (nspidStr== NULL || *nspidStr == '\0') + return -1; + + pid = strtol(nspidStr, &endptr, 10); + if (*endptr != '\0') + bail("unable to parse _NSLISTENERPID"); + + return pid; +} + +void nsexec(void) +{ + int nsPid, nsFd; + char *path; + + nsPid = getNslistenerPid(); + if (nsPid <= 0) { + return; + } + + path = malloc(sizeof(char)*64); + sprintf(path, "/proc/%d/ns/net", nsPid); + nsFd = open(path, O_RDONLY); + setns(nsFd, CLONE_NEWNET); + + free(path); + close(nsFd); + return; +} diff --git a/shim.go b/shim.go index 6f7ce13d..a57eeb74 100644 --- a/shim.go +++ b/shim.go @@ -65,6 +65,26 @@ var shimCommand = cli.Command{ defer signal.Stop(sigc) } + stateDir := filepath.Join(root, container) + if _, err = os.Stat(stateDir); err == nil { + // state dir exist, write shim pid into it + shimFile := filepath.Join(stateDir, "shim-"+process+".pid") + f, err := os.OpenFile(shimFile, os.O_WRONLY|os.O_CREATE, 0640) + if err != nil { + glog.Errorf("can't create shim pid file: %v", err) + goto eventHandle + } + defer f.Close() + shimPid := fmt.Sprintf("%d", os.Getpid()) + _, err = f.Write([]byte(shimPid)) + if err != nil { + glog.Errorf("can't write pid %q to shim pid file: %v", shimPid, err) + goto eventHandle + } + defer os.Remove(shimFile) + } + + eventHandle: // wait until exit evChan := containerEvents(c, container) for e := range evChan { diff --git a/spec.go b/spec.go index c6713ce3..4fb8018e 100644 --- a/spec.go +++ b/spec.go @@ -103,3 +103,23 @@ func loadSpec(ocffile string) (*specs.Spec, error) { } return &spec, nil } + +func updateSpec(spec *specs.Spec, ocffile string) error { + if _, err := os.Stat(ocffile); err != nil { + if os.IsNotExist(err) { + return fmt.Errorf("%q doesn't exists", ocffile) + } + return fmt.Errorf("Stat %q error: %v", ocffile, err) + } + + data, err := json.Marshal(spec) + if err != nil { + return fmt.Errorf("failed to marshal spec file during update: %v", err) + } + + if err = ioutil.WriteFile(ocffile, data, 0640); err != nil { + return err + } + + return nil +}