Merge pull request #5827 from mlsorensen/pvc-enospc

Allow for configurable minimum PVC reserve bytes, default to 128k

api/openapi-spec/swagger.json

@@ -9427,6 +9427,10 @@
       "type": "integer",
       "format": "int32"
      },
+     "minimumReservePVCBytes": {
+      "type": "integer",
+      "format": "int64"
+     },
      "nodeSelectors": {
       "type": "object",
       "additionalProperties": {

cmd/virt-launcher/virt-launcher.go

@@ -341,6 +341,7 @@ func main() {
 	hookSidecars := pflag.Uint("hook-sidecars", 0, "Number of requested hook sidecars, virt-launcher will wait for all of them to become available")
 	noFork := pflag.Bool("no-fork", false, "Fork and let virt-launcher watch itself to react to crashes if set to false")
 	lessPVCSpaceToleration := pflag.Int("less-pvc-space-toleration", 0, "Toleration in percent when PVs' available space is smaller than requested")
+	minimumPVCReserveBytes := pflag.Uint64("minimum-pvc-reserve-bytes", 131072, "Minimum reserve to keep empty on PVC during auto-provision of disk.img")
 	ovmfPath := pflag.String("ovmf-path", "/usr/share/OVMF", "The directory that contains the EFI roms (like OVMF_CODE.fd)")
 	qemuAgentSysInterval := pflag.Duration("qemu-agent-sys-interval", 120, "Interval in seconds between consecutive qemu agent calls for sys commands")
 	qemuAgentFileInterval := pflag.Duration("qemu-agent-file-interval", 300, "Interval in seconds between consecutive qemu agent calls for file command")
@@ -410,7 +411,7 @@ func main() {
 	notifier := notifyclient.NewNotifier(*virtShareDir)
 	defer notifier.Close()
 
-	domainManager, err := virtwrap.NewLibvirtDomainManager(domainConn, *virtShareDir, notifier, *lessPVCSpaceToleration, &agentStore, *ovmfPath, ephemeralDiskCreator)
+	domainManager, err := virtwrap.NewLibvirtDomainManager(domainConn, *virtShareDir, notifier, *lessPVCSpaceToleration, *minimumPVCReserveBytes, &agentStore, *ovmfPath, ephemeralDiskCreator)
 	if err != nil {
 		panic(err)
 	}

manifests/generated/kv-resource.yaml

@@ -130,6 +130,9 @@ spec:
                         type: object
                       memoryOvercommit:
                         type: integer
+                      minimumReservePVCBytes:
+                        format: int64
+                        type: integer
                       nodeSelectors:
                         additionalProperties:
                           type: string
@@ -1991,6 +1994,9 @@ spec:
                         type: object
                       memoryOvercommit:
                         type: integer
+                      minimumReservePVCBytes:
+                        format: int64
+                        type: integer
                       nodeSelectors:
                         additionalProperties:
                           type: string

pkg/host-disk/host-disk.go

@@ -97,13 +97,13 @@ func ReplacePVCByHostDisk(vmi *v1.VirtualMachineInstance, clientset kubecli.Kube
 	return nil
 }
 
-func dirBytesAvailable(path string) (uint64, error) {
+func dirBytesAvailable(path string, reserve uint64) (uint64, error) {
 	var stat syscall.Statfs_t
 	err := syscall.Statfs(path, &stat)
 	if err != nil {
 		return 0, err
 	}
-	return stat.Bavail * uint64(stat.Bsize), nil
+	return stat.Bavail*uint64(stat.Bsize) - reserve, nil
 }
 
 func createSparseRaw(fullPath string, size int64) (err error) {
@@ -133,20 +133,22 @@ func GetMountedHostDiskDir(volumeName string) string {
 }
 
 type DiskImgCreator struct {
-	dirBytesAvailableFunc  func(path string) (uint64, error)
+	dirBytesAvailableFunc  func(path string, reserve uint64) (uint64, error)
 	notifier               k8sNotifier
 	lessPVCSpaceToleration int
+	minimumPVCReserveBytes uint64
 }
 
 type k8sNotifier interface {
 	SendK8sEvent(vmi *v1.VirtualMachineInstance, severity string, reason string, message string) error
 }
 
-func NewHostDiskCreator(notifier k8sNotifier, lessPVCSpaceToleration int) DiskImgCreator {
+func NewHostDiskCreator(notifier k8sNotifier, lessPVCSpaceToleration int, minimumPVCReserveBytes uint64) DiskImgCreator {
 	return DiskImgCreator{
 		dirBytesAvailableFunc:  dirBytesAvailable,
 		notifier:               notifier,
 		lessPVCSpaceToleration: lessPVCSpaceToleration,
+		minimumPVCReserveBytes: minimumPVCReserveBytes,
 	}
 }
 
@@ -190,7 +192,7 @@ func (hdc *DiskImgCreator) mountHostDiskAndSetOwnership(vmi *v1.VirtualMachineIn
 }
 
 func (hdc *DiskImgCreator) handleRequestedSizeAndCreateSparseRaw(vmi *v1.VirtualMachineInstance, diskDir string, diskPath string, hostDisk *v1.HostDisk) error {
-	size, err := hdc.dirBytesAvailableFunc(diskDir)
+	size, err := hdc.dirBytesAvailableFunc(diskDir, hdc.minimumPVCReserveBytes)
 	availableSize := int64(size)
 	if err != nil {
 		return err

pkg/host-disk/host-disk_test.go

@@ -115,7 +115,8 @@ var _ = Describe("HostDisk", func() {
 	notifier := MockNotifier{
 		Events: make(chan k8sv1.Event, 10),
 	}
-	hostDiskCreator := NewHostDiskCreator(notifier, 0)
+	hostDiskCreator := NewHostDiskCreator(notifier, 0, 0)
+	hostDiskCreatorWithReserve := NewHostDiskCreator(notifier, 10, 1048576)
 
 	Describe("HostDisk with 'Disk' type", func() {
 		It("Should not create a disk.img when it exists", func() {
@@ -210,6 +211,74 @@ var _ = Describe("HostDisk", func() {
 					Expect(true).To(Equal(os.IsNotExist(err)))
 				})
 
+				It("Should NOT subtract reserve if there is enough space on storage for requested size", func(done Done) {
+					By("Creating a new minimal vmi")
+					vmi := v1.NewMinimalVMI("fake-vmi")
+
+					By("Adding HostDisk volumes")
+					addHostDisk(vmi, "volume1", v1.HostDiskExistsOrCreate, "64Mi")
+
+					By("Executing CreateHostDisks func which should create a full-size disk.img")
+					err := hostDiskCreatorWithReserve.Create(vmi)
+					Expect(err).NotTo(HaveOccurred())
+
+					img1, err := os.Stat(vmi.Spec.Volumes[0].HostDisk.Path)
+					Expect(err).NotTo(HaveOccurred())
+					Expect(img1.Size()).To(Equal(int64(67108864))) // 64Mi
+					close(done)
+				}, 5)
+
+				It("Should subtract reserve if there is NOT enough space on storage for requested size", func(done Done) {
+					By("Creating a new minimal vmi")
+					vmi := v1.NewMinimalVMI("fake-vmi")
+					dirAvailable := uint64(64 << 20)
+
+					hostDiskCreatorWithReserve.dirBytesAvailableFunc = func(path string, reserve uint64) (uint64, error) {
+						return dirAvailable - reserve, nil
+					}
+
+					By("Adding HostDisk volume that is slightly too large for available bytes when reserve is accounted for")
+					addHostDisk(vmi, "volume1", v1.HostDiskExistsOrCreate, "64Mi")
+
+					By("Executing CreateHostDisks func which should create disk.img minus reserve")
+					err := hostDiskCreatorWithReserve.Create(vmi)
+					Expect(err).NotTo(HaveOccurred())
+
+					img1, err := os.Stat(vmi.Spec.Volumes[0].HostDisk.Path)
+					Expect(err).NotTo(HaveOccurred())
+					Expect(img1.Size()).To(BeNumerically("==", dirAvailable-hostDiskCreatorWithReserve.minimumPVCReserveBytes)) // 64Mi minus reserve
+					close(done)
+				}, 5)
+
+				It("Should refuse to create disk image if reserve causes image to exceed lessPVCSpaceToleration", func(done Done) {
+					By("Creating a new minimal vmi")
+					vmi := v1.NewMinimalVMI("fake-vmi")
+					dirAvailable := uint64(64 << 20)
+
+					hostDiskCreatorWithReserve.dirBytesAvailableFunc = func(path string, reserve uint64) (uint64, error) {
+						return dirAvailable - reserve, nil
+					}
+					hostDiskCreatorWithReserve.setlessPVCSpaceToleration(1) // 1% of 64Mi, tolerate up to 671088 bytes lost
+
+					By("Adding HostDisk volume that is slightly too large for available bytes when reserve is accounted for")
+					addHostDisk(vmi, "volume1", v1.HostDiskExistsOrCreate, "64Mi")
+
+					By("Executing CreateHostDisks func which should NOT create disk.img minus reserve")
+					err := hostDiskCreatorWithReserve.Create(vmi)
+					Expect(err).To(HaveOccurred())
+
+					_, err = os.Stat(vmi.Spec.Volumes[0].HostDisk.Path)
+					Expect(true).To(Equal(os.IsNotExist(err)))
+
+					event := <-notifier.Events
+					Expect(event.InvolvedObject.Namespace).To(Equal(vmi.Namespace))
+					Expect(event.InvolvedObject.Name).To(Equal(vmi.Name))
+					Expect(event.Type).To(Equal(EventTypeToleratedSmallPV))
+					Expect(event.Reason).To(Equal(EventReasonToleratedSmallPV))
+					Expect(event.Message).To(ContainSubstring("PV size too small"))
+					close(done)
+				}, 5)
+
 				It("Should take lessPVCSpaceToleration into account when creating disk images", func(done Done) {
 					By("Creating a new minimal vmi")
 					vmi := v1.NewMinimalVMI("fake-vmi")
@@ -222,7 +291,7 @@ var _ = Describe("HostDisk", func() {
 						return origSize * (100 - uint64(toleration) + uint64(diff)) / 100
 					}
 
-					fakeDirBytesAvailable := func(path string) (uint64, error) {
+					fakeDirBytesAvailable := func(path string, reserve uint64) (uint64, error) {
 						if strings.Contains(path, "volume1") {
 							// toleration +1
 							return calcToleratedSize(size64Mi, 1), nil

pkg/virt-config/config-map.go

@@ -186,6 +186,7 @@ func defaultClusterConfig() *v1.KubeVirtConfiguration {
 			UseEmulation:           DefaultUseEmulation,
 			MemoryOvercommit:       DefaultMemoryOvercommit,
 			LessPVCSpaceToleration: DefaultLessPVCSpaceToleration,
+			MinimumReservePVCBytes: DefaultMinimumReservePVCBytes,
 			NodeSelectors:          nodeSelectorsDefault,
 			CPUAllocationRatio:     DefaultCPUAllocationRatio,
 			LogVerbosity: &v1.LogVerbosity{

pkg/virt-config/config_test.go

@@ -457,7 +457,7 @@ var _ = Describe("ConfigMap", func() {
 			func(c *v1.KubeVirtConfiguration) interface{} {
 				return c.DeveloperConfiguration
 			},
-			`{"featureGates":["test1","test2"],"pvcTolerateLessSpaceUpToPercent":5,"memoryOvercommit":150,"nodeSelectors":{"test":"test"},"useEmulation":true,"cpuAllocationRatio":25,"logVerbosity":{"virtAPI":2,"virtController":2,"virtHandler":2,"virtLauncher":2,"virtOperator":2}}`),
+			`{"featureGates":["test1","test2"],"pvcTolerateLessSpaceUpToPercent":5,"minimumReservePVCBytes":131072,"memoryOvercommit":150,"nodeSelectors":{"test":"test"},"useEmulation":true,"cpuAllocationRatio":25,"logVerbosity":{"virtAPI":2,"virtController":2,"virtHandler":2,"virtLauncher":2,"virtOperator":2}}`),
 		table.Entry("when networkConfiguration set, should equal to result",
 			v1.KubeVirtConfiguration{
 				NetworkConfiguration: &v1.NetworkConfiguration{

pkg/virt-config/virt-config.go

@@ -49,6 +49,7 @@ const (
 	DefaultPPC64LEEmulatedMachines                  = "pseries*"
 	DefaultAARCH64EmulatedMachines                  = "virt*"
 	DefaultLessPVCSpaceToleration                   = 10
+	DefaultMinimumReservePVCBytes                   = 131072
 	DefaultNodeSelectors                            = ""
 	DefaultNetworkInterface                         = "bridge"
 	DefaultImagePullPolicy                          = k8sv1.PullIfNotPresent
@@ -163,6 +164,10 @@ func (c *ClusterConfig) GetLessPVCSpaceToleration() int {
 	return c.GetConfig().DeveloperConfiguration.LessPVCSpaceToleration
 }
 
+func (c *ClusterConfig) GetMinimumReservePVCBytes() uint64 {
+	return c.GetConfig().DeveloperConfiguration.MinimumReservePVCBytes
+}
+
 func (c *ClusterConfig) GetNodeSelectors() map[string]string {
 	return c.GetConfig().DeveloperConfiguration.NodeSelectors
 }

pkg/virt-controller/services/template.go

@@ -957,6 +957,7 @@ func (t *templateService) renderLaunchManifest(vmi *v1.VirtualMachineInstance, t
 	}
 
 	lessPVCSpaceToleration := t.clusterConfig.GetLessPVCSpaceToleration()
+	reservePVCBytes := t.clusterConfig.GetMinimumReservePVCBytes()
 	ovmfPath := t.clusterConfig.GetOVMFPath()
 
 	var command []string
@@ -978,6 +979,7 @@ func (t *templateService) renderLaunchManifest(vmi *v1.VirtualMachineInstance, t
 			"--grace-period-seconds", strconv.Itoa(int(gracePeriodSeconds)),
 			"--hook-sidecars", strconv.Itoa(len(requestedHookSidecarList)),
 			"--less-pvc-space-toleration", strconv.Itoa(lessPVCSpaceToleration),
+			"--minimum-pvc-reserve-bytes", strconv.FormatUint(reservePVCBytes, 10),
 			"--ovmf-path", ovmfPath,
 		}
 	}