Skip to content

Commit

Permalink
Add SELinux context tracking to volume manager
Browse files Browse the repository at this point in the history 
Both ActualStateOfWorld and DesiredStateOfWorld must track SELinux context
of volume mounts.
  • Loading branch information
@jsafrane
jsafrane committed Aug 4, 2022
1 parent 4cfb277 commit 48b0751
Show file tree
Hide file tree
Showing 13 changed files with 421 additions and 93 deletions.
121 changes: 103 additions & 18 deletions pkg/kubelet/volumemanager/cache/actual_state_of_world.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,9 @@ import (
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
"k8s.io/apimachinery/pkg/types"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/klog/v2"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/volume"
"k8s.io/kubernetes/pkg/volume/util"
"k8s.io/kubernetes/pkg/volume/util/operationexecutor"
Expand Down Expand Up @@ -73,7 +75,7 @@ type ActualStateOfWorld interface {
// global mount point prior to detach.
// If a volume with the name volumeName does not exist in the list of
// attached volumes, an error is returned.
SetDeviceMountState(volumeName v1.UniqueVolumeName, deviceMountState operationexecutor.DeviceMountState, devicePath, deviceMountPath string) error
SetDeviceMountState(volumeName v1.UniqueVolumeName, deviceMountState operationexecutor.DeviceMountState, devicePath, deviceMountPath, seLinuxMountContext string) error

// DeletePodFromVolume removes the given pod from the given volume in the
// cache indicating the volume has been successfully unmounted from the pod.
Expand Down Expand Up @@ -107,7 +109,7 @@ type ActualStateOfWorld interface {
// volumes, depend on this to update the contents of the volume.
// All volume mounting calls should be idempotent so a second mount call for
// volumes that do not need to update contents should not fail.
PodExistsInVolume(podName volumetypes.UniquePodName, volumeName v1.UniqueVolumeName, desiredVolumeSize resource.Quantity) (bool, string, error)
PodExistsInVolume(podName volumetypes.UniquePodName, volumeName v1.UniqueVolumeName, desiredVolumeSize resource.Quantity, seLinuxLabel string) (bool, string, error)

// PodRemovedFromVolume returns true if the given pod does not exist in the list of
// mountedPods for the given volume in the cache, indicating that the pod has
Expand Down Expand Up @@ -182,6 +184,11 @@ type AttachedVolume struct {

// DeviceMountState indicates if device has been globally mounted or is not.
DeviceMountState operationexecutor.DeviceMountState

// SELinuxMountContext is the context with that the volume is globally mounted
// (via -o context=XYZ mount option). If empty, the volume is not mounted with
// "-o context=".
SELinuxMountContext string
}

// DeviceMayBeMounted returns true if device is mounted in global path or is in
Expand Down Expand Up @@ -288,6 +295,11 @@ type attachedVolume struct {
// persistentVolumeSize records size of the volume when pod was started or
// size after successful completion of volume expansion operation.
persistentVolumeSize *resource.Quantity

// seLinuxMountContext is the context with that the volume is mounted to global directory
// (via -o context=XYZ mount option). If nil, the volume is not mounted. If "", the volume is
// mounted without "-o context=".
seLinuxMountContext *string
}

// The mountedPod object represents a pod for which the kubelet volume manager
Expand Down Expand Up @@ -333,6 +345,11 @@ type mountedPod struct {
// - VolumeMounted: means volume for pod has been successfully mounted
// - VolumeMountUncertain: means volume for pod may not be mounted, but it must be unmounted
volumeMountStateForPod operationexecutor.VolumeMountState

// seLinuxMountContext is the context with that the volume is mounted to Pod directory
// (via -o context=XYZ mount option). If nil, the volume is not mounted. If "", the volume is
// mounted without "-o context=".
seLinuxMountContext string
}

func (asw *actualStateOfWorld) MarkVolumeAsAttached(
Expand Down Expand Up @@ -465,13 +482,13 @@ func (asw *actualStateOfWorld) MarkVolumeAsUnmounted(
}

func (asw *actualStateOfWorld) MarkDeviceAsMounted(
volumeName v1.UniqueVolumeName, devicePath, deviceMountPath string) error {
return asw.SetDeviceMountState(volumeName, operationexecutor.DeviceGloballyMounted, devicePath, deviceMountPath)
volumeName v1.UniqueVolumeName, devicePath, deviceMountPath, seLinuxMountContext string) error {
return asw.SetDeviceMountState(volumeName, operationexecutor.DeviceGloballyMounted, devicePath, deviceMountPath, seLinuxMountContext)
}

func (asw *actualStateOfWorld) MarkDeviceAsUncertain(
volumeName v1.UniqueVolumeName, devicePath, deviceMountPath string) error {
return asw.SetDeviceMountState(volumeName, operationexecutor.DeviceMountUncertain, devicePath, deviceMountPath)
volumeName v1.UniqueVolumeName, devicePath, deviceMountPath, seLinuxMountContext string) error {
return asw.SetDeviceMountState(volumeName, operationexecutor.DeviceMountUncertain, devicePath, deviceMountPath, seLinuxMountContext)
}

func (asw *actualStateOfWorld) MarkVolumeMountAsUncertain(markVolumeOpts operationexecutor.MarkVolumeOpts) error {
Expand All @@ -481,7 +498,7 @@ func (asw *actualStateOfWorld) MarkVolumeMountAsUncertain(markVolumeOpts operati

func (asw *actualStateOfWorld) MarkDeviceAsUnmounted(
volumeName v1.UniqueVolumeName) error {
return asw.SetDeviceMountState(volumeName, operationexecutor.DeviceNotMounted, "", "")
return asw.SetDeviceMountState(volumeName, operationexecutor.DeviceNotMounted, "", "", "")
}

func (asw *actualStateOfWorld) GetDeviceMountState(volumeName v1.UniqueVolumeName) operationexecutor.DeviceMountState {
Expand Down Expand Up @@ -629,6 +646,7 @@ func (asw *actualStateOfWorld) AddPodToVolume(markVolumeOpts operationexecutor.M
volumeGidValue: volumeGidValue,
volumeSpec: volumeSpec,
volumeMountStateForPod: markVolumeOpts.VolumeMountState,
seLinuxMountContext: markVolumeOpts.SELinuxMountContext,
}
}

Expand All @@ -646,6 +664,15 @@ func (asw *actualStateOfWorld) AddPodToVolume(markVolumeOpts operationexecutor.M
podObj.mounter = mounter
}
asw.attachedVolumes[volumeName].mountedPods[podName] = podObj
if utilfeature.DefaultFeatureGate.Enabled(features.SELinuxMountReadWriteOncePod) {
// Store the mount context also in the AttachedVolume to have a global volume context
// for a quick comparison in PodExistsInVolume.
if volumeObj.seLinuxMountContext == nil {
volumeObj.seLinuxMountContext = &markVolumeOpts.SELinuxMountContext
asw.attachedVolumes[volumeName] = volumeObj
}
}

return nil
}

Expand Down Expand Up @@ -685,7 +712,7 @@ func (asw *actualStateOfWorld) MarkRemountRequired(
}

func (asw *actualStateOfWorld) SetDeviceMountState(
volumeName v1.UniqueVolumeName, deviceMountState operationexecutor.DeviceMountState, devicePath, deviceMountPath string) error {
volumeName v1.UniqueVolumeName, deviceMountState operationexecutor.DeviceMountState, devicePath, deviceMountPath, seLinuxMountContext string) error {
asw.Lock()
defer asw.Unlock()

Expand All @@ -701,6 +728,11 @@ func (asw *actualStateOfWorld) SetDeviceMountState(
if devicePath != "" {
volumeObj.devicePath = devicePath
}
if utilfeature.DefaultFeatureGate.Enabled(features.SELinuxMountReadWriteOncePod) {
if seLinuxMountContext != "" {
volumeObj.seLinuxMountContext = &seLinuxMountContext
}
}
asw.attachedVolumes[volumeName] = volumeObj
return nil
}
Expand Down Expand Up @@ -776,7 +808,7 @@ func (asw *actualStateOfWorld) DeleteVolume(volumeName v1.UniqueVolumeName) erro
return nil
}

func (asw *actualStateOfWorld) PodExistsInVolume(podName volumetypes.UniquePodName, volumeName v1.UniqueVolumeName, desiredVolumeSize resource.Quantity) (bool, string, error) {
func (asw *actualStateOfWorld) PodExistsInVolume(podName volumetypes.UniquePodName, volumeName v1.UniqueVolumeName, desiredVolumeSize resource.Quantity, seLinuxLabel string) (bool, string, error) {
asw.RLock()
defer asw.RUnlock()

Expand All @@ -785,6 +817,22 @@ func (asw *actualStateOfWorld) PodExistsInVolume(podName volumetypes.UniquePodNa
return false, "", newVolumeNotAttachedError(volumeName)
}

if utilfeature.DefaultFeatureGate.Enabled(features.SELinuxMountReadWriteOncePod) {
if volumeObj.seLinuxMountContext != nil {
// The volume is mounted, check its SELinux context mount option
if *volumeObj.seLinuxMountContext != seLinuxLabel {
fullErr := newSELinuxMountMismatchError(volumeName)
if util.IsRWOP(volumeObj.spec) {
return false, volumeObj.devicePath, fullErr
} else {
// This is not an error yet, but it will be when support for RWO and RWX volumes is added
// TODO: bump some metric here
klog.V(4).ErrorS(fullErr, "Please report this error in https://github.com/kubernetes/enhancements/issues/1710, together with full Pod yaml file")
}
}
}
}

podObj, podExists := volumeObj.mountedPods[podName]
if podExists {
// if volume mount was uncertain we should keep trying to mount the volume
Expand Down Expand Up @@ -905,7 +953,6 @@ func (asw *actualStateOfWorld) GetAllMountedVolumes() []MountedVolume {
mountedVolume,
getMountedVolume(&podObj, &volumeObj))
}

}
}

Expand Down Expand Up @@ -1010,15 +1057,22 @@ func (asw *actualStateOfWorld) SyncReconstructedVolume(volumeName v1.UniqueVolum

func (asw *actualStateOfWorld) newAttachedVolume(
attachedVolume *attachedVolume) AttachedVolume {
seLinuxMountContext := ""
if utilfeature.DefaultFeatureGate.Enabled(features.SELinuxMountReadWriteOncePod) {
if attachedVolume.seLinuxMountContext != nil {
seLinuxMountContext = *attachedVolume.seLinuxMountContext
}
}
return AttachedVolume{
AttachedVolume: operationexecutor.AttachedVolume{
VolumeName: attachedVolume.volumeName,
VolumeSpec: attachedVolume.spec,
NodeName: asw.nodeName,
PluginIsAttachable: attachedVolume.pluginIsAttachable,
DevicePath: attachedVolume.devicePath,
DeviceMountPath: attachedVolume.deviceMountPath,
PluginName: attachedVolume.pluginName},
VolumeName: attachedVolume.volumeName,
VolumeSpec: attachedVolume.spec,
NodeName: asw.nodeName,
PluginIsAttachable: attachedVolume.pluginIsAttachable,
DevicePath: attachedVolume.devicePath,
DeviceMountPath: attachedVolume.deviceMountPath,
PluginName: attachedVolume.pluginName,
SELinuxMountContext: seLinuxMountContext},
DeviceMountState: attachedVolume.deviceMountState,
}
}
Expand Down Expand Up @@ -1105,6 +1159,10 @@ func IsFSResizeRequiredError(err error) bool {
// mountedPod and attachedVolume objects.
func getMountedVolume(
mountedPod *mountedPod, attachedVolume *attachedVolume) MountedVolume {
seLinuxMountContext := ""
if attachedVolume.seLinuxMountContext != nil {
seLinuxMountContext = *attachedVolume.seLinuxMountContext
}
return MountedVolume{
MountedVolume: operationexecutor.MountedVolume{
PodName: mountedPod.podName,
Expand All @@ -1117,5 +1175,32 @@ func getMountedVolume(
BlockVolumeMapper: mountedPod.blockVolumeMapper,
VolumeGidValue: mountedPod.volumeGidValue,
VolumeSpec: mountedPod.volumeSpec,
DeviceMountPath: attachedVolume.deviceMountPath}}
DeviceMountPath: attachedVolume.deviceMountPath,
SELinuxMountContext: seLinuxMountContext}}

}

// seLinuxMountMismatchError is an error returned when PodExistsInVolume() found
// a volume mounted with a different SELinux label than expected.
type seLinuxMountMismatchError struct {
volumeName v1.UniqueVolumeName
}

func (err seLinuxMountMismatchError) Error() string {
return fmt.Sprintf(
"volumeName %q is already mounted to a different pod with a different SELinux label",
err.volumeName)
}

func newSELinuxMountMismatchError(volumeName v1.UniqueVolumeName) error {
return seLinuxMountMismatchError{
volumeName: volumeName,
}
}

// IsSELinuxMountMismatchError returns true if the specified error is a
// seLinuxMountMismatchError.
func IsSELinuxMountMismatchError(err error) bool {
_, ok := err.(seLinuxMountMismatchError)
return ok
}
8 changes: 4 additions & 4 deletions pkg/kubelet/volumemanager/cache/actual_state_of_world_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -737,7 +737,7 @@ func Test_MarkDeviceAsMounted_Positive_NewVolume(t *testing.T) {
}

// Act
err = asw.MarkDeviceAsMounted(generatedVolumeName, devicePath, deviceMountPath)
err = asw.MarkDeviceAsMounted(generatedVolumeName, devicePath, deviceMountPath, "")

// Assert
if err != nil {
Expand Down Expand Up @@ -824,7 +824,7 @@ func TestUncertainVolumeMounts(t *testing.T) {
t.Fatalf("expected volume %s to be found in aws.GetPossiblyMountedVolumesForPod", volumeSpec1.Name())
}

volExists, _, _ := asw.PodExistsInVolume(podName1, generatedVolumeName1, resource.Quantity{})
volExists, _, _ := asw.PodExistsInVolume(podName1, generatedVolumeName1, resource.Quantity{}, "")
if volExists {
t.Fatalf("expected volume %s to not exist in asw", generatedVolumeName1)
}
Expand Down Expand Up @@ -910,7 +910,7 @@ func verifyPodExistsInVolumeAsw(
expectedDevicePath string,
asw ActualStateOfWorld) {
podExistsInVolume, devicePath, err :=
asw.PodExistsInVolume(expectedPodName, expectedVolumeName, resource.Quantity{})
asw.PodExistsInVolume(expectedPodName, expectedVolumeName, resource.Quantity{}, "")
if err != nil {
t.Fatalf(
"ASW PodExistsInVolume failed. Expected: <no error> Actual: <%v>", err)
Expand Down Expand Up @@ -952,7 +952,7 @@ func verifyPodDoesntExistInVolumeAsw(
expectVolumeToExist bool,
asw ActualStateOfWorld) {
podExistsInVolume, devicePath, err :=
asw.PodExistsInVolume(podToCheck, volumeToCheck, resource.Quantity{})
asw.PodExistsInVolume(podToCheck, volumeToCheck, resource.Quantity{}, "")
if !expectVolumeToExist && err == nil {
t.Fatalf(
"ASW PodExistsInVolume did not return error. Expected: <error indicating volume does not exist> Actual: <%v>", err)
Expand Down

0 comments on commit 48b0751

Please sign in to comment.