From 21047d80d5dca040f9df07b4ede3b819757c2104 Mon Sep 17 00:00:00 2001 From: Zhang Wei Date: Fri, 30 Jun 2017 17:24:20 +0800 Subject: [PATCH 1/3] Join shim into nslistener network ns runv shim should share same network namespace with nslistener, then if user try to insert an network interface into runv shim's netns, nslistener can observe the changes and copy the configure into VM. Signed-off-by: Zhang Wei --- containerd/api/grpc/server/server.go | 1 + create.go | 36 +++++++++++++++---- exec.go | 6 ++-- nsenter/namespace.h | 14 ++++++++ nsenter/nsenter.go | 12 +++++++ nsenter/nsenter_gccgo.go | 25 +++++++++++++ nsenter/nsenter_unsupported.go | 5 +++ nsenter/nsexec.c | 54 ++++++++++++++++++++++++++++ 8 files changed, 143 insertions(+), 10 deletions(-) create mode 100644 nsenter/namespace.h create mode 100644 nsenter/nsenter.go create mode 100644 nsenter/nsenter_gccgo.go create mode 100644 nsenter/nsenter_unsupported.go create mode 100644 nsenter/nsexec.c diff --git a/containerd/api/grpc/server/server.go b/containerd/api/grpc/server/server.go index fbf5175e..c24ee32e 100644 --- a/containerd/api/grpc/server/server.go +++ b/containerd/api/grpc/server/server.go @@ -232,6 +232,7 @@ func supervisorContainer2ApiContainer(c *supervisor.Container) *types.Container BundlePath: c.BundlePath, Status: "running", Runtime: "runv", + Labels: []string{fmt.Sprintf("nslistener=%d", c.Processes["init"].ProcId)}, } } diff --git a/create.go b/create.go index d7224322..d926454c 100644 --- a/create.go +++ b/create.go @@ -6,11 +6,13 @@ import ( "os" "os/exec" "path/filepath" + "strconv" "strings" "syscall" "time" "github.com/hyperhq/runv/containerd/api/grpc/types" + _ "github.com/hyperhq/runv/nsenter" "github.com/kardianos/osext" "github.com/kr/pty" "github.com/opencontainers/runtime-spec/specs-go" @@ -238,6 +240,8 @@ func checkConsole(context *cli.Context, p *specs.Process, createOnly bool) error // * In runv, shared namespaces multiple containers are located in the same VM which is managed by a runv-daemon. // * Any running container can exit in any arbitrary order, the runv-daemon and the VM are existed until the last container of the VM is existed +type createFunction func(stdin, stdout, stderr string) (int, error) + func createContainer(context *cli.Context, container, namespace string, config *specs.Spec) error { address := filepath.Join(namespace, "namespaced.sock") c, err := getClient(address) @@ -245,7 +249,7 @@ func createContainer(context *cli.Context, container, namespace string, config * return err } - return ociCreate(context, container, "init", func(stdin, stdout, stderr string) error { + return ociCreate(context, container, "init", func(stdin, stdout, stderr string) (int, error) { r := &types.CreateContainerRequest{ Id: container, Runtime: "runv-create", @@ -255,21 +259,23 @@ func createContainer(context *cli.Context, container, namespace string, config * Stderr: stderr, } - if _, err := c.CreateContainer(netcontext.Background(), r); err != nil { - return err + ctr, err := c.CreateContainer(netcontext.Background(), r) + if err != nil { + return -1, err } + nslPID := retrieveNslPID(ctr.Container.Labels) // create symbol link to namespace file namespaceDir := filepath.Join(context.GlobalString("root"), container, "namespace") if err := os.Symlink(namespace, namespaceDir); err != nil { - return fmt.Errorf("failed to create symbol link %q: %v", filepath.Join(namespaceDir, "namespaced.sock"), err) + return -1, fmt.Errorf("failed to create symbol link %q: %v", filepath.Join(namespaceDir, "namespaced.sock"), err) } - return nil + return nslPID, nil }) } -func ociCreate(context *cli.Context, container, process string, createFunc func(stdin, stdout, stderr string) error) error { +func ociCreate(context *cli.Context, container, process string, createFunc createFunction) error { path, err := osext.Executable() if err != nil { return fmt.Errorf("cannot find self executable path for %s: %v\n", os.Args[0], err) @@ -303,7 +309,7 @@ func ociCreate(context *cli.Context, container, process string, createFunc func( stdout = tty.Name() stderr = tty.Name() } - err = createFunc(stdin, stdout, stderr) + listenerPID, err := createFunc(stdin, stdout, stderr) if err != nil { return err } @@ -338,6 +344,9 @@ func ociCreate(context *cli.Context, container, process string, createFunc func( Setsid: true, }, } + if listenerPID > 0 { + cmd.Env = []string{fmt.Sprintf("_NSLISTENERPID=%d", listenerPID)} + } err = cmd.Start() if err != nil { return err @@ -372,3 +381,16 @@ func createPidFile(path string, pid int) error { } return os.Rename(tmpName, path) } + +func retrieveNslPID(labels []string) int { + for _, v := range labels { + s := strings.SplitN(v, "=", 2) + if len(s) == 2 && s[0] == "nslistener" { + pid, err := strconv.Atoi(s[1]) + if err == nil { + return pid + } + } + } + return -1 +} diff --git a/exec.go b/exec.go index a4f3ab48..29a02e1b 100644 --- a/exec.go +++ b/exec.go @@ -208,7 +208,7 @@ func runProcess(context *cli.Context, container string, config *specs.Process) ( monitorTtySize(c, container, process) } - err = ociCreate(context, container, process, func(stdin, stdout, stderr string) error { + err = ociCreate(context, container, process, func(stdin, stdout, stderr string) (int, error) { p := &types.AddProcessRequest{ Id: container, Pid: process, @@ -225,9 +225,9 @@ func runProcess(context *cli.Context, container string, config *specs.Process) ( Stderr: stderr, } if _, err := c.AddProcess(netcontext.Background(), p); err != nil { - return err + return -1, err } - return nil + return -1, nil }) if err != nil { return -1, err diff --git a/nsenter/namespace.h b/nsenter/namespace.h new file mode 100644 index 00000000..d62ffd36 --- /dev/null +++ b/nsenter/namespace.h @@ -0,0 +1,14 @@ +#ifndef NSENTER_NAMESPACE_H +#define NSENTER_NAMESPACE_H + +#ifndef _GNU_SOURCE +# define _GNU_SOURCE +#endif +#include + +/* All of these are taken from include/uapi/linux/sched.h */ +#ifndef CLONE_NEWNET +# define CLONE_NEWNET 0x40000000 /* New network namespace */ +#endif + +#endif /* NSENTER_NAMESPACE_H */ diff --git a/nsenter/nsenter.go b/nsenter/nsenter.go new file mode 100644 index 00000000..07f4d63e --- /dev/null +++ b/nsenter/nsenter.go @@ -0,0 +1,12 @@ +// +build linux,!gccgo + +package nsenter + +/* +#cgo CFLAGS: -Wall +extern void nsexec(); +void __attribute__((constructor)) init(void) { + nsexec(); +} +*/ +import "C" diff --git a/nsenter/nsenter_gccgo.go b/nsenter/nsenter_gccgo.go new file mode 100644 index 00000000..63c7a3ec --- /dev/null +++ b/nsenter/nsenter_gccgo.go @@ -0,0 +1,25 @@ +// +build linux,gccgo + +package nsenter + +/* +#cgo CFLAGS: -Wall +extern void nsexec(); +void __attribute__((constructor)) init(void) { + nsexec(); +} +*/ +import "C" + +// AlwaysFalse is here to stay false +// (and be exported so the compiler doesn't optimize out its reference) +var AlwaysFalse bool + +func init() { + if AlwaysFalse { + // by referencing this C init() in a noop test, it will ensure the compiler + // links in the C function. + // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=65134 + C.init() + } +} diff --git a/nsenter/nsenter_unsupported.go b/nsenter/nsenter_unsupported.go new file mode 100644 index 00000000..ac701ca3 --- /dev/null +++ b/nsenter/nsenter_unsupported.go @@ -0,0 +1,5 @@ +// +build !linux !cgo + +package nsenter + +import "C" diff --git a/nsenter/nsexec.c b/nsenter/nsexec.c new file mode 100644 index 00000000..f6b3464b --- /dev/null +++ b/nsenter/nsexec.c @@ -0,0 +1,54 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include + +/* Get all of the CLONE_NEW* flags. */ +#include "namespace.h" + +#define bail(fmt, ...) \ + do { \ + int ret = __COUNTER__ + 1; \ + fprintf(stderr, "nsenter: " fmt ": %m\n", ##__VA_ARGS__); \ + exit(ret); \ + } while(0) + +static int getNslistenerPid(void) +{ + int pid; + char *nspidStr, *endptr; + + nspidStr = getenv("_NSLISTENERPID"); + if (nspidStr== NULL || *nspidStr == '\0') + return -1; + + pid = strtol(nspidStr, &endptr, 10); + if (*endptr != '\0') + bail("unable to parse _NSLISTENERPID"); + + return pid; +} + +void nsexec(void) +{ + int nsPid, nsFd; + char *path; + + nsPid = getNslistenerPid(); + if (nsPid <= 0) { + return; + } + + path = malloc(sizeof(char)*64); + sprintf(path, "/proc/%d/ns/net", nsPid); + nsFd = open(path, O_RDONLY); + setns(nsFd, CLONE_NEWNET); + + free(path); + close(nsFd); + return; +} From cfd46953f14fc483792d9076b4c4df30e94a7ff7 Mon Sep 17 00:00:00 2001 From: Zhang Wei Date: Fri, 23 Jun 2017 17:27:09 +0800 Subject: [PATCH 2/3] Enable pod support Share a pod with existing container, with `--net container:xxx`. Signed-off-by: Zhang Wei --- create.go | 59 ++++++++++++++++++++++++++++++++++++++++++++++++++----- list.go | 5 ++--- shim.go | 20 +++++++++++++++++++ spec.go | 20 +++++++++++++++++++ 4 files changed, 96 insertions(+), 8 deletions(-) diff --git a/create.go b/create.go index d926454c..ea097467 100644 --- a/create.go +++ b/create.go @@ -6,6 +6,7 @@ import ( "os" "os/exec" "path/filepath" + "regexp" "strconv" "strings" "syscall" @@ -99,16 +100,25 @@ func runContainer(context *cli.Context, createOnly bool) error { sharedContainer = spec.Annotations["ocid/sandbox_name"] } } else { - for _, ns := range spec.Linux.Namespaces { + for i, ns := range spec.Linux.Namespaces { if ns.Path != "" { - if strings.Contains(ns.Path, "/") { - return fmt.Errorf("Runv doesn't support path to namespace file, it supports containers name as shared namespaces only") - } if ns.Type == "mount" { // TODO support it! return fmt.Errorf("Runv doesn't support shared mount namespace currently") } - sharedContainer = ns.Path + if sharedContainer, err = findSharedContainer(context.GlobalString("root"), ns.Path); err != nil { + return fmt.Errorf("failed to find shared container: %v", err) + } + + cstate, err := getContainer(context, sharedContainer) + if err != nil { + return fmt.Errorf("can't get state file for container %q: %v", sharedContainer, err) + } + spec.Linux.Namespaces[i] = specs.LinuxNamespace{ + Type: ns.Type, + Path: fmt.Sprintf("/proc/%d/ns/%s", cstate.InitProcessPid, ns.Type), + } + _, err = os.Stat(filepath.Join(root, sharedContainer, stateJson)) if err != nil { return fmt.Errorf("The container %q is not existing or not ready", sharedContainer) @@ -117,6 +127,10 @@ func runContainer(context *cli.Context, createOnly bool) error { if err != nil { return fmt.Errorf("The container %q is not ready", sharedContainer) } + + if err = updateSpec(spec, ocffile); err != nil { + return fmt.Errorf("update spec file failed: %v", err) + } } } } @@ -352,6 +366,7 @@ func ociCreate(context *cli.Context, container, process string, createFunc creat return err } } + if context.String("pid-file") != "" { err = createPidFile(context.String("pid-file"), cmd.Process.Pid) if err != nil { @@ -394,3 +409,37 @@ func retrieveNslPID(labels []string) int { } return -1 } + +func findSharedContainer(root, nsPath string) (container string, err error) { + absRoot, err := filepath.Abs(root) + if err != nil { + return "", err + } + list, err := ioutil.ReadDir(absRoot) + if err != nil { + return "", err + } + + if strings.Contains(nsPath, "/") { + pidexp := regexp.MustCompile(`/proc/(\d+)/ns/*`) + matches := pidexp.FindStringSubmatch(nsPath) + if len(matches) != 2 { + return "", fmt.Errorf("malformed ns path: %s", nsPath) + } + pid := matches[1] + + for _, item := range list { + shimPidFile := filepath.Join(absRoot, item.Name(), "shim-init.pid") + spidByte, err := ioutil.ReadFile(shimPidFile) + if err != nil { + return "", fmt.Errorf("failed to read shim pid file %q: %v", shimPidFile, err) + } + spid := strings.TrimSpace(string(spidByte)) + if pid == spid { + return item.Name(), nil + } + } + return "", fmt.Errorf("can't find container with shim pid %s", pid) + } + return nsPath, nil +} diff --git a/list.go b/list.go index 5aabc4a4..24c82387 100644 --- a/list.go +++ b/list.go @@ -50,7 +50,7 @@ in json format: }, }, Action: func(context *cli.Context) { - s, err := getContainers(context) + s, err := getContainers(context.GlobalString("root")) if err != nil { fatal(err) } @@ -91,8 +91,7 @@ in json format: }, } -func getContainers(context *cli.Context) ([]containerState, error) { - root := context.GlobalString("root") +func getContainers(root string) ([]containerState, error) { absRoot, err := filepath.Abs(root) if err != nil { return nil, err diff --git a/shim.go b/shim.go index 6f7ce13d..a57eeb74 100644 --- a/shim.go +++ b/shim.go @@ -65,6 +65,26 @@ var shimCommand = cli.Command{ defer signal.Stop(sigc) } + stateDir := filepath.Join(root, container) + if _, err = os.Stat(stateDir); err == nil { + // state dir exist, write shim pid into it + shimFile := filepath.Join(stateDir, "shim-"+process+".pid") + f, err := os.OpenFile(shimFile, os.O_WRONLY|os.O_CREATE, 0640) + if err != nil { + glog.Errorf("can't create shim pid file: %v", err) + goto eventHandle + } + defer f.Close() + shimPid := fmt.Sprintf("%d", os.Getpid()) + _, err = f.Write([]byte(shimPid)) + if err != nil { + glog.Errorf("can't write pid %q to shim pid file: %v", shimPid, err) + goto eventHandle + } + defer os.Remove(shimFile) + } + + eventHandle: // wait until exit evChan := containerEvents(c, container) for e := range evChan { diff --git a/spec.go b/spec.go index c6713ce3..4fb8018e 100644 --- a/spec.go +++ b/spec.go @@ -103,3 +103,23 @@ func loadSpec(ocffile string) (*specs.Spec, error) { } return &spec, nil } + +func updateSpec(spec *specs.Spec, ocffile string) error { + if _, err := os.Stat(ocffile); err != nil { + if os.IsNotExist(err) { + return fmt.Errorf("%q doesn't exists", ocffile) + } + return fmt.Errorf("Stat %q error: %v", ocffile, err) + } + + data, err := json.Marshal(spec) + if err != nil { + return fmt.Errorf("failed to marshal spec file during update: %v", err) + } + + if err = ioutil.WriteFile(ocffile, data, 0640); err != nil { + return err + } + + return nil +} From 81d42e43151cce44dd02957a03cb50e2bcaf0561 Mon Sep 17 00:00:00 2001 From: Zhang Wei Date: Mon, 31 Jul 2017 14:55:41 +0800 Subject: [PATCH 3/3] bugfix: can't create pod due to miss of shim-init.pid Old version runv won't create shim-init.pid, so for handling backward compatibility problem, ignore the old dir for composing POD. Signed-off-by: Zhang Wei --- create.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/create.go b/create.go index ea097467..0f63c440 100644 --- a/create.go +++ b/create.go @@ -432,7 +432,9 @@ func findSharedContainer(root, nsPath string) (container string, err error) { shimPidFile := filepath.Join(absRoot, item.Name(), "shim-init.pid") spidByte, err := ioutil.ReadFile(shimPidFile) if err != nil { - return "", fmt.Errorf("failed to read shim pid file %q: %v", shimPidFile, err) + // for backward compatibility, if dir doesn't contain "shim-init.pid" + // it could be old legacy dir, ignore and skip it. + continue } spid := strings.TrimSpace(string(spidByte)) if pid == spid {