use sidecar container to execute rbd command

api/swagger-spec/v1.json

@@ -12262,6 +12262,10 @@
      "readOnly": {
       "type": "boolean",
       "description": "ReadOnly here will force the ReadOnly setting in VolumeMounts. Defaults to false. More info: http://releases.k8s.io/HEAD/examples/rbd/README.md#how-to-use-it"
+     },
+     "sidecar": {
+      "type": "string",
+      "description": "Optional: Sidecar is the sidecar container's name"
      }
     }
    },
@@ -12765,6 +12769,14 @@
      "tty": {
       "type": "boolean",
       "description": "Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. Default is false."
+     },
+     "stdout": {
+      "type": "boolean",
+      "description": "Whether this container should send stdout. Default is false."
+     },
+     "stderr": {
+      "type": "boolean",
+      "description": "Whether this container should send stderr. Default is false."
      }
     }
    },

examples/rbd/README.md

@@ -51,6 +51,7 @@ Once you have installed Ceph and new Kubernetes, you can create a pod based on m
 - *secretName*: The name of the authentication secrets. If provided, *secretName* overrides *keyring*. Note, see below about how to create a secret.
 - *fsType*: The filesystem type (ext4, xfs, etc) that formatted on the device.
 - *readOnly*: Whether the filesystem is used as readOnly.
+- *sidecar*: The name of the sidecar container that can run *rbd* command.
 
 # Use Ceph Authentication Secret
 

pkg/api/deep_copy_generated.go

@@ -238,6 +238,8 @@ func deepCopy_api_Container(in Container, out *Container, c *conversion.Cloner)
 	}
 	out.Stdin = in.Stdin
 	out.TTY = in.TTY
+	out.Stdout = in.Stdout
+	out.Stderr = in.Stderr
 	return nil
 }
 
@@ -1568,6 +1570,7 @@ func deepCopy_api_RBDVolumeSource(in RBDVolumeSource, out *RBDVolumeSource, c *c
 		out.SecretRef = nil
 	}
 	out.ReadOnly = in.ReadOnly
+	out.Sidecar = in.Sidecar
 	return nil
 }
 

pkg/api/types.go

@@ -568,6 +568,8 @@ type RBDVolumeSource struct {
 	// Optional: Defaults to false (read/write). ReadOnly here will force
 	// the ReadOnly setting in VolumeMounts.
 	ReadOnly bool `json:"readOnly,omitempty"`
+	// Optional: Sidecar is the sidecar container's name
+	Sidecar string `json:"sidecar,omitempty"`
 }
 
 // CinderVolumeSource represents a cinder volume resource in Openstack.
@@ -793,6 +795,12 @@ type Container struct {
 	// and shouldn't be used for general purpose containers.
 	Stdin bool `json:"stdin,omitempty"`
 	TTY   bool `json:"tty,omitempty"`
+	// Whether this container should send stdout.
+	// Default is false.
+	Stdout bool `json:"stdout,omitempty"`
+	// Whether this container should send stderr.
+	// Default is false.
+	Stderr bool `json:"stderr,omitempty"`
 }
 
 // Handler defines a specific action that should be taken

pkg/api/v1/conversion_generated.go

@@ -261,6 +261,8 @@ func convert_api_Container_To_v1_Container(in *api.Container, out *Container, s
 	}
 	out.Stdin = in.Stdin
 	out.TTY = in.TTY
+	out.Stdout = in.Stdout
+	out.Stderr = in.Stderr
 	return nil
 }
 
@@ -1745,6 +1747,7 @@ func convert_api_RBDVolumeSource_To_v1_RBDVolumeSource(in *api.RBDVolumeSource,
 		out.SecretRef = nil
 	}
 	out.ReadOnly = in.ReadOnly
+	out.Sidecar = in.Sidecar
 	return nil
 }
 
@@ -2663,6 +2666,8 @@ func convert_v1_Container_To_api_Container(in *Container, out *api.Container, s
 	}
 	out.Stdin = in.Stdin
 	out.TTY = in.TTY
+	out.Stdout = in.Stdout
+	out.Stderr = in.Stderr
 	return nil
 }
 
@@ -4147,6 +4152,7 @@ func convert_v1_RBDVolumeSource_To_api_RBDVolumeSource(in *RBDVolumeSource, out
 		out.SecretRef = nil
 	}
 	out.ReadOnly = in.ReadOnly
+	out.Sidecar = in.Sidecar
 	return nil
 }
 

pkg/api/v1/deep_copy_generated.go

@@ -253,6 +253,8 @@ func deepCopy_v1_Container(in Container, out *Container, c *conversion.Cloner) e
 	}
 	out.Stdin = in.Stdin
 	out.TTY = in.TTY
+	out.Stdout = in.Stdout
+	out.Stderr = in.Stderr
 	return nil
 }
 
@@ -1568,6 +1570,7 @@ func deepCopy_v1_RBDVolumeSource(in RBDVolumeSource, out *RBDVolumeSource, c *co
 		out.SecretRef = nil
 	}
 	out.ReadOnly = in.ReadOnly
+	out.Sidecar = in.Sidecar
 	return nil
 }
 

pkg/api/v1/types.go

@@ -582,6 +582,8 @@ type RBDVolumeSource struct {
 	// Defaults to false.
 	// More info: http://releases.k8s.io/HEAD/examples/rbd/README.md#how-to-use-it
 	ReadOnly bool `json:"readOnly,omitempty"`
+	// Optional: Sidecar is the sidecar container's name
+	Sidecar string `json:"sidecar,omitempty"`
 }
 
 // CinderVolumeSource represents a cinder volume resource in Openstack.
@@ -988,6 +990,12 @@ type Container struct {
 	// Whether this container should allocate a TTY for itself, also requires 'stdin' to be true.
 	// Default is false.
 	TTY bool `json:"tty,omitempty"`
+	// Whether this container should send stdout.
+	// Default is false.
+	Stdout bool `json:"stdout,omitempty"`
+	// Whether this container should send stderr.
+	// Default is false.
+	Stderr bool `json:"stderr,omitempty"`
 }
 
 // Handler defines a specific action that should be taken

pkg/api/v1/types_swagger_doc_generated.go

@@ -144,6 +144,8 @@ var map_Container = map[string]string{
 	"securityContext":        "Security options the pod should run with. More info: http://releases.k8s.io/HEAD/docs/design/security_context.md",
 	"stdin":                  "Whether this container should allocate a buffer for stdin in the container runtime. Default is false.",
 	"tty":                    "Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. Default is false.",
+	"stdout":                 "Whether this container should send stdout. Default is false.",
+	"stderr":                 "Whether this container should send stderr. Default is false.",
 }
 
 func (Container) SwaggerDoc() map[string]string {
@@ -1036,6 +1038,7 @@ var map_RBDVolumeSource = map[string]string{
 	"keyring":   "Keyring is the path to key ring for RBDUser. Default is /etc/ceph/keyring. More info: http://releases.k8s.io/HEAD/examples/rbd/README.md#how-to-use-it",
 	"secretRef": "SecretRef is name of the authentication secret for RBDUser. If provided overrides keyring. Default is empty. More info: http://releases.k8s.io/HEAD/examples/rbd/README.md#how-to-use-it",
 	"readOnly":  "ReadOnly here will force the ReadOnly setting in VolumeMounts. Defaults to false. More info: http://releases.k8s.io/HEAD/examples/rbd/README.md#how-to-use-it",
+	"sidecar":   "Optional: Sidecar is the sidecar container's name",
 }
 
 func (RBDVolumeSource) SwaggerDoc() map[string]string {

pkg/controller/persistentvolume/persistentvolume_recycler_controller.go

@@ -233,3 +233,7 @@ func (f *PersistentVolumeRecycler) NewWrapperCleaner(spec *volume.Spec, podUID t
 func (f *PersistentVolumeRecycler) GetCloudProvider() cloudprovider.Interface {
 	return nil
 }
+
+func (f *PersistentVolumeRecycler) RunContainerCommand(pod *api.Pod, container *api.Container, cmd []string) ([]byte, error) {
+	return nil, fmt.Errorf("RunInContainer not supported by PVClaimBinder's VolumeHost implementation")
+}

pkg/expapi/deep_copy_generated.go

@@ -175,6 +175,8 @@ func deepCopy_api_Container(in api.Container, out *api.Container, c *conversion.
 	}
 	out.Stdin = in.Stdin
 	out.TTY = in.TTY
+	out.Stdout = in.Stdout
+	out.Stderr = in.Stderr
 	return nil
 }
 
@@ -520,6 +522,7 @@ func deepCopy_api_RBDVolumeSource(in api.RBDVolumeSource, out *api.RBDVolumeSour
 		out.SecretRef = nil
 	}
 	out.ReadOnly = in.ReadOnly
+	out.Sidecar = in.Sidecar
 	return nil
 }
 

pkg/expapi/v1/conversion_generated.go

@@ -190,6 +190,8 @@ func convert_api_Container_To_v1_Container(in *api.Container, out *v1.Container,
 	}
 	out.Stdin = in.Stdin
 	out.TTY = in.TTY
+	out.Stdout = in.Stdout
+	out.Stderr = in.Stderr
 	return nil
 }
 
@@ -547,6 +549,7 @@ func convert_api_RBDVolumeSource_To_v1_RBDVolumeSource(in *api.RBDVolumeSource,
 		out.SecretRef = nil
 	}
 	out.ReadOnly = in.ReadOnly
+	out.Sidecar = in.Sidecar
 	return nil
 }
 
@@ -957,6 +960,8 @@ func convert_v1_Container_To_api_Container(in *v1.Container, out *api.Container,
 	}
 	out.Stdin = in.Stdin
 	out.TTY = in.TTY
+	out.Stdout = in.Stdout
+	out.Stderr = in.Stderr
 	return nil
 }
 
@@ -1314,6 +1319,7 @@ func convert_v1_RBDVolumeSource_To_api_RBDVolumeSource(in *v1.RBDVolumeSource, o
 		out.SecretRef = nil
 	}
 	out.ReadOnly = in.ReadOnly
+	out.Sidecar = in.Sidecar
 	return nil
 }
 

pkg/expapi/v1/deep_copy_generated.go

@@ -192,6 +192,8 @@ func deepCopy_v1_Container(in v1.Container, out *v1.Container, c *conversion.Clo
 	}
 	out.Stdin = in.Stdin
 	out.TTY = in.TTY
+	out.Stdout = in.Stdout
+	out.Stderr = in.Stderr
 	return nil
 }
 
@@ -538,6 +540,7 @@ func deepCopy_v1_RBDVolumeSource(in v1.RBDVolumeSource, out *v1.RBDVolumeSource,
 		out.SecretRef = nil
 	}
 	out.ReadOnly = in.ReadOnly
+	out.Sidecar = in.Sidecar
 	return nil
 }
 

pkg/kubelet/container/fake_runtime.go

@@ -312,3 +312,11 @@ func (f *FakeRuntime) PortForward(pod *Pod, port uint16, stream io.ReadWriteClos
 	f.CalledFunctions = append(f.CalledFunctions, "PortForward")
 	return f.Err
 }
+
+func (f *FakeRuntime) RunContainerCommand(pod *api.Pod, container *api.Container, cmd []string) ([]byte, error) {
+	f.Lock()
+	defer f.Unlock()
+
+	f.CalledFunctions = append(f.CalledFunctions, "RunContainerCommand")
+	return nil, f.Err
+}

pkg/kubelet/container/runtime.go

@@ -103,6 +103,8 @@ type ContainerCommandRunner interface {
 	ExecInContainer(containerID string, cmd []string, stdin io.Reader, stdout, stderr io.WriteCloser, tty bool) error
 	// Forward the specified port from the specified pod to the stream.
 	PortForward(pod *Pod, port uint16, stream io.ReadWriteCloser) error
+	// Runs the command in the container
+	RunContainerCommand(pod *api.Pod, container *api.Container, cmd []string) ([]byte, error)
 }
 
 // ImagePuller wraps Runtime.PullImage() to pull a container image.

pkg/kubelet/dockertools/manager.go

@@ -663,8 +663,10 @@ func (dm *DockerManager) runContainer(
 			WorkingDir: container.WorkingDir,
 			Labels:     labels,
 			// Interactive containers:
-			OpenStdin: container.Stdin,
-			Tty:       container.TTY,
+			OpenStdin:    container.Stdin,
+			Tty:          container.TTY,
+			AttachStdout: container.Stdout,
+			AttachStderr: container.Stderr,
 		},
 	}
 
@@ -1871,3 +1873,35 @@ func (dm *DockerManager) doBackOff(pod *api.Pod, container *api.Container, podSt
 	dm.clearReasonCache(pod, container)
 	return false
 }
+
+// create and start a container then log the output
+func (dm *DockerManager) RunContainerCommand(pod *api.Pod, container *api.Container, cmd []string) ([]byte, error) {
+	opts, err := dm.generator.GenerateRunContainerOptions(pod, container)
+	if err != nil {
+		return nil, err
+	}
+
+	utsMode := ""
+	if pod.Spec.HostNetwork {
+		utsMode = "host"
+	}
+	netNamespace := ""
+	if pod.Spec.HostNetwork {
+		netNamespace = "host"
+	}
+
+	// create and start container
+	// see https://docs.docker.com/reference/api/docker_remote_api_v1.20/#3-1-inside-docker-run
+	id, err := dm.runContainer(pod, container, opts, nil, netNamespace, "", utsMode)
+	if err != nil {
+		return nil, err
+	}
+	defer dm.KillContainerInPod("", container, pod)
+
+	var stdout, stderr bytes.Buffer
+	if err := dm.GetContainerLogs(pod, id, "all", true, &stdout, &stderr); err != nil {
+		return nil, err
+	}
+	out := append(stdout.Bytes(), stderr.Bytes()...)
+	return out, nil
+}

pkg/kubelet/kubelet.go

@@ -2786,3 +2786,17 @@ func extractBandwidthResources(pod *api.Pod) (ingress, egress *resource.Quantity
 	}
 	return ingress, egress, nil
 }
+
+// Runs the command in the container
+func (kl *Kubelet) RunContainerCommand(pod *api.Pod, container *api.Container, cmd []string) ([]byte, error) {
+	// Mount volumes.
+	podFullName := kubecontainer.GetPodFullName(pod)
+	podVolumes, err := kl.mountExternalVolumes(pod)
+	if err != nil {
+		err = fmt.Errorf("Unable to mount volumes for pod %q: %v; skipping pod", podFullName, err)
+		return nil, err
+	}
+	kl.volumeManager.SetVolumes(pod.UID, podVolumes)
+
+	return kl.runner.RunContainerCommand(pod, container, cmd)
+}

pkg/kubelet/kubelet_test.go

@@ -797,6 +797,10 @@ func (f *fakeContainerCommandRunner) PortForward(pod *kubecontainer.Pod, port ui
 	return nil
 }
 
+func (f *fakeContainerCommandRunner) RunContainerCommand(pod *api.Pod, container *api.Container, cmd []string) ([]byte, error) {
+	return nil, f.E
+}
+
 func TestRunInContainerNoSuchPod(t *testing.T) {
 	testKubelet := newTestKubelet(t)
 	kubelet := testKubelet.kubelet

pkg/kubelet/lifecycle/handlers_test.go

@@ -91,6 +91,10 @@ func (f *fakeContainerCommandRunner) PortForward(pod *kubecontainer.Pod, port ui
 	return nil
 }
 
+func (f *fakeContainerCommandRunner) RunContainerCommand(pod *api.Pod, container *api.Container, cmd []string) ([]byte, error) {
+	return []byte{}, nil
+}
+
 func TestRunHandlerExec(t *testing.T) {
 	fakeCommandRunner := fakeContainerCommandRunner{}
 	handlerRunner := NewHandlerRunner(&fakeHTTP{}, &fakeCommandRunner, nil)

pkg/kubelet/rkt/rkt.go

@@ -1143,6 +1143,11 @@ func (r *runtime) ExecInContainer(containerID string, cmd []string, stdin io.Rea
 	return command.Run()
 }
 
+//TODO placeholder
+func (r *runtime) RunContainerCommand(pod *api.Pod, container *api.Container, cmd []string) ([]byte, error) {
+	return nil, fmt.Errorf("not supported")
+}
+
 // findRktID returns the rkt uuid for the pod.
 func (r *runtime) findRktID(pod *kubecontainer.Pod) (string, error) {
 	serviceName := makePodServiceFileName(pod.ID)

pkg/kubelet/volumes.go

@@ -84,6 +84,10 @@ func (vh *volumeHost) GetCloudProvider() cloudprovider.Interface {
 	return vh.kubelet.cloud
 }
 
+func (vh *volumeHost) RunContainerCommand(pod *api.Pod, container *api.Container, cmd []string) ([]byte, error) {
+	return vh.kubelet.RunContainerCommand(pod, container, cmd)
+}
+
 func (kl *Kubelet) newVolumeBuilderFromPlugins(spec *volume.Spec, pod *api.Pod, opts volume.VolumeOptions, mounter mount.Interface) (volume.Builder, error) {
 	plugin, err := kl.volumePluginMgr.FindPluginBySpec(spec)
 	if err != nil {

pkg/volume/plugins.go

@@ -125,6 +125,9 @@ type VolumeHost interface {
 
 	//Get cloud provider from kubelet
 	GetCloudProvider() cloudprovider.Interface
+
+	// Runs the command in the container
+	RunContainerCommand(pod *api.Pod, container *api.Container, cmd []string) ([]byte, error)
 }
 
 // VolumePluginMgr tracks registered plugins.