Merge pull request #237 from sameo/topic/sandbox_netns

Enable networking for hypervisor based container runtimes

oci/oci.go

@@ -18,6 +18,7 @@ import (
 
 	"github.com/Sirupsen/logrus"
 	"github.com/kubernetes-incubator/cri-o/utils"
+	"github.com/containernetworking/cni/pkg/ns"
 	"golang.org/x/sys/unix"
 	"k8s.io/kubernetes/pkg/fields"
 	pb "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime"
@@ -344,6 +345,7 @@ type Container struct {
 	annotations fields.Set
 	image       *pb.ImageSpec
 	sandbox     string
+	netns       ns.NetNS
 	terminal    bool
 	state       *ContainerState
 	metadata    *pb.ContainerMetadata
@@ -360,14 +362,15 @@ type ContainerState struct {
 }
 
 // NewContainer creates a container object.
-func NewContainer(id string, name string, bundlePath string, logPath string, labels map[string]string, annotations map[string]string, image *pb.ImageSpec, metadata *pb.ContainerMetadata, sandbox string, terminal bool) (*Container, error) {
+func NewContainer(id string, name string, bundlePath string, logPath string, netns ns.NetNS, labels map[string]string, annotations map[string]string, image *pb.ImageSpec, metadata *pb.ContainerMetadata, sandbox string, terminal bool) (*Container, error) {
 	c := &Container{
 		id:          id,
 		name:        name,
 		bundlePath:  bundlePath,
 		logPath:     logPath,
 		labels:      labels,
 		sandbox:     sandbox,
+		netns:       netns,
 		terminal:    terminal,
 		metadata:    metadata,
 		annotations: annotations,
@@ -421,7 +424,12 @@ func (c *Container) NetNsPath() (string, error) {
 	if c.state == nil {
 		return "", fmt.Errorf("container state is not populated")
 	}
-	return fmt.Sprintf("/proc/%d/ns/net", c.state.Pid), nil
+
+	if c.netns == nil {
+		return fmt.Sprintf("/proc/%d/ns/net", c.state.Pid), nil
+	}
+
+	return c.netns.Path(), nil
 }
 
 // Metadata returns the metadata of the container.

server/container_create.go

@@ -273,14 +273,20 @@ func (s *Server) createSandboxContainer(containerID string, containerName string
 
 	logrus.Debugf("pod container state %+v", podInfraState)
 
-	for nsType, nsFile := range map[string]string{
-		"ipc":     "ipc",
-		"network": "net",
-	} {
-		nsPath := fmt.Sprintf("/proc/%d/ns/%s", podInfraState.Pid, nsFile)
-		if err := specgen.AddOrReplaceLinuxNamespace(nsType, nsPath); err != nil {
-			return nil, err
-		}
+	ipcNsPath := fmt.Sprintf("/proc/%d/ns/ipc", podInfraState.Pid)
+	if err := specgen.AddOrReplaceLinuxNamespace("ipc", ipcNsPath); err != nil {
+		return nil, err
+	}
+
+	netNsPath := sb.netNsPath()
+	if netNsPath == "" {
+		// The sandbox does not have a permanent namespace,
+		// it's on the host one.
+		netNsPath = fmt.Sprintf("/proc/%d/ns/net", podInfraState.Pid)
+	}
+
+	if err := specgen.AddOrReplaceLinuxNamespace("network", netNsPath); err != nil {
+		return nil, err
 	}
 
 	imageSpec := containerConfig.GetImage()
@@ -336,7 +342,7 @@ func (s *Server) createSandboxContainer(containerID string, containerName string
 		return nil, err
 	}
 
-	container, err := oci.NewContainer(containerID, containerName, containerDir, logPath, labels, annotations, imageSpec, metadata, sb.id, containerConfig.GetTty())
+	container, err := oci.NewContainer(containerID, containerName, containerDir, logPath, sb.netNs(), labels, annotations, imageSpec, metadata, sb.id, containerConfig.GetTty())
 	if err != nil {
 		return nil, err
 	}

server/sandbox.go

@@ -1,15 +1,128 @@
 package server
 
 import (
+	"crypto/rand"
 	"errors"
 	"fmt"
+	"os"
+	"path/filepath"
+	"sync"
 
+	"github.com/Sirupsen/logrus"
 	"github.com/docker/docker/pkg/stringid"
 	"github.com/kubernetes-incubator/cri-o/oci"
+	"github.com/containernetworking/cni/pkg/ns"
 	"k8s.io/kubernetes/pkg/fields"
 	pb "k8s.io/kubernetes/pkg/kubelet/api/v1alpha1/runtime"
 )
 
+type sandboxNetNs struct {
+	sync.Mutex
+	ns      ns.NetNS
+	symlink *os.File
+	closed  bool
+}
+
+func (ns *sandboxNetNs) symlinkCreate(name string) error {
+	b := make([]byte, 4)
+	_, randErr := rand.Reader.Read(b)
+	if randErr != nil {
+		return randErr
+	}
+
+	nsName := fmt.Sprintf("%s-%x", name, b)
+	symlinkPath := filepath.Join(nsRunDir, nsName)
+
+	if err := os.Symlink(ns.ns.Path(), symlinkPath); err != nil {
+		return err
+	}
+
+	fd, err := os.Open(symlinkPath)
+	if err != nil {
+		if removeErr := os.RemoveAll(symlinkPath); removeErr != nil {
+			return removeErr
+		}
+
+		return err
+	}
+
+	ns.symlink = fd
+
+	return nil
+}
+
+func (ns *sandboxNetNs) symlinkRemove() error {
+	if err := ns.symlink.Close(); err != nil {
+		return err
+	}
+
+	return os.RemoveAll(ns.symlink.Name())
+}
+
+func isSymbolicLink(path string) (bool, error) {
+	fi, err := os.Lstat(path)
+	if err != nil {
+		return false, err
+	}
+
+	return fi.Mode()&os.ModeSymlink == os.ModeSymlink, nil
+}
+
+func netNsGet(nspath, name string) (*sandboxNetNs, error) {
+	if err := ns.IsNSorErr(nspath); err != nil {
+		return nil, errSandboxClosedNetNS
+	}
+
+	symlink, symlinkErr := isSymbolicLink(nspath)
+	if symlinkErr != nil {
+		return nil, symlinkErr
+	}
+
+	var resolvedNsPath string
+	if symlink {
+		path, err := os.Readlink(nspath)
+		if err != nil {
+			return nil, err
+		}
+		resolvedNsPath = path
+	} else {
+		resolvedNsPath = nspath
+	}
+
+	netNS, err := ns.GetNS(resolvedNsPath)
+	if err != nil {
+		return nil, err
+	}
+
+	netNs := &sandboxNetNs{ns: netNS, closed: false,}
+
+	if symlink {
+		fd, err := os.Open(nspath)
+		if err != nil {
+			return nil, err
+		}
+
+		netNs.symlink = fd
+	} else {
+		if err := netNs.symlinkCreate(name); err != nil {
+			return nil, err
+		}
+	}
+
+	return netNs, nil
+}
+
+func hostNetNsPath() (string, error) {
+	netNS, err := ns.GetCurrentNS()
+	if err != nil {
+		return "", err
+	}
+
+	defer netNS.Close()
+
+	return netNS.Path(), nil
+}
+
 type sandbox struct {
 	id             string
 	name           string
@@ -20,17 +133,20 @@ type sandbox struct {
 	containers     oci.Store
 	processLabel   string
 	mountLabel     string
+	netns          *sandboxNetNs
 	metadata       *pb.PodSandboxMetadata
 	shmPath        string
 }
 
 const (
 	podDefaultNamespace = "default"
 	defaultShmSize      = 64 * 1024 * 1024
+	nsRunDir            = "/var/run/netns"
 )
 
 var (
-	errSandboxIDEmpty = errors.New("PodSandboxId should not be empty")
+	errSandboxIDEmpty     = errors.New("PodSandboxId should not be empty")
+	errSandboxClosedNetNS = errors.New("PodSandbox networking namespace is closed")
 )
 
 func (s *sandbox) addContainer(c *oci.Container) {
@@ -45,6 +161,77 @@ func (s *sandbox) removeContainer(c *oci.Container) {
 	s.containers.Delete(c.Name())
 }
 
+func (s *sandbox) netNs() ns.NetNS {
+	if s.netns == nil {
+		return nil
+	}
+
+	return s.netns.ns
+}
+
+func (s *sandbox) netNsPath() string {
+	if s.netns == nil {
+		return ""
+	}
+
+	return s.netns.symlink.Name()
+}
+
+func (s *sandbox) netNsCreate() error {
+	if s.netns != nil {
+		return fmt.Errorf("net NS already created")
+	}
+
+	netNS, err := ns.NewNS()
+	if err != nil {
+		return err
+	}
+
+	s.netns = &sandboxNetNs{
+		ns: netNS,
+		closed: false,
+	}
+
+	if err := s.netns.symlinkCreate(s.name); err != nil {
+		logrus.Warnf("Could not create nentns symlink %v", err)
+
+		if err := s.netns.ns.Close(); err != nil {
+			return err
+		}
+
+		return err
+	}
+
+	return nil
+}
+
+func (s *sandbox) netNsRemove() error {
+	if s.netns == nil {
+		logrus.Warn("no networking namespace")
+		return nil
+	}
+
+	s.netns.Lock()
+	defer s.netns.Unlock()
+
+	if s.netns.closed {
+		// netNsRemove() can be called multiple
+		// times without returning an error.
+		return nil
+	}
+
+	if err := s.netns.symlinkRemove(); err != nil {
+		return err
+	}
+
+	if err := s.netns.ns.Close(); err != nil {
+		return err
+	}
+
+	s.netns.closed = true
+	return nil
+}
+
 func (s *Server) generatePodIDandName(name string, namespace string, attempt uint32) (string, string, error) {
 	var (
 		err error

server/sandbox_remove.go

@@ -73,6 +73,10 @@ func (s *Server) RemovePodSandbox(ctx context.Context, req *pb.RemovePodSandboxR
 		}
 	}
 
+	if err := sb.netNsRemove(); err != nil {
+		return nil, fmt.Errorf("failed to remove networking namespace for sandbox %s: %v", sb.id, err)
+	}
+
 	// Remove the files related to the sandbox
 	podSandboxDir := filepath.Join(s.config.SandboxDir, sb.id)
 	if err := os.RemoveAll(podSandboxDir); err != nil {