Optional space reservation (thin provisioning)

README.md

@@ -65,6 +65,7 @@ parameters:
   hostname: storage-1.domain.tld
   type: hostpath
   node: storage-1 # the kubernetes.io/hostname label if different than hostname parameter (optional)
+  reserveSpace: true
 ```
 
 Following example configures a storage class for ZFS over [NFS][nfs]:
@@ -80,6 +81,7 @@ parameters:
   hostname: storage-1.domain.tld
   type: nfs
   shareProperties: rw,no_root_squash # no_root_squash by default sets mode to 'ro'
+  reserveSpace: true
 ```
 For NFS, you can also specify other options, as described in [exports(5)][man exports].
 
@@ -94,11 +96,12 @@ further significant to the provisioner.
 
 ### Storage space
 
-The provisioner uses the `refreservation` and `refquota` ZFS attributes to limit
-storage space for volumes. Each volume can not use more storage space than
-the given resource request and also reserves exactly that much. This means
-that over provisioning is not possible. Snapshots **do not** account for the
-storage space limit, however this provisioner does not do any snapshots or backups.
+By default, the provisioner uses the `refreservation` and `refquota` ZFS attributes
+to limit storage space for volumes. Each volume can not use more storage space than
+the given resource request and also reserves exactly that much. To disable this and
+enable thin provisioning, set `reserveSpace` to `false` in your storage class parameters.
+Snapshots **do not** account for the storage space limit, however this provisioner
+does not do any snapshots or backups.
 
 See [zfs(8)][man zfs] for more information.
 

go.mod

@@ -4,8 +4,8 @@ go 1.19
 
 require (
 	github.com/knadh/koanf v1.4.3
-	github.com/mistifyio/go-zfs v2.1.1+incompatible
 	github.com/stretchr/testify v1.8.0
+	github.com/mistifyio/go-zfs/v3 v3.0.1
 	k8s.io/api v0.24.3
 	k8s.io/apimachinery v0.24.3
 	k8s.io/client-go v0.24.3
@@ -30,7 +30,7 @@ require (
 	github.com/google/gnostic v0.5.7-v3refs // indirect
 	github.com/google/go-cmp v0.5.7 // indirect
 	github.com/google/gofuzz v1.1.0 // indirect
-	github.com/google/uuid v1.1.2 // indirect
+	github.com/google/uuid v1.2.0 // indirect
 	github.com/imdario/mergo v0.3.5 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect

go.sum

@@ -219,6 +219,8 @@ github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLe
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs=
+github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
@@ -317,6 +319,8 @@ github.com/miekg/dns v1.1.41 h1:WMszZWJG0XmzbK9FEmzH2TVcqYzFesusSIB41b8KHxY=
 github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
 github.com/mistifyio/go-zfs v2.1.1+incompatible h1:gAMO1HM9xBRONLHHYnu5iFsOJUiJdNZo6oqSENd4eW8=
 github.com/mistifyio/go-zfs v2.1.1+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4=
+github.com/mistifyio/go-zfs/v3 v3.0.1 h1:YaoXgBePoMA12+S1u/ddkv+QqxcfiZK4prI6HPnkFiU=
+github.com/mistifyio/go-zfs/v3 v3.0.1/go.mod h1:CzVgeB0RvF2EGzQnytKVvVSDwmKJXxkOTUGbNrTja/k=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
 github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI=
 github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=

pkg/provisioner/delete_test.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"testing"
 
-	gozfs "github.com/mistifyio/go-zfs"
+	gozfs "github.com/mistifyio/go-zfs/v3"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	core "k8s.io/api/core/v1"

pkg/provisioner/parameters.go

@@ -11,6 +11,7 @@ const (
 	HostnameParameter        = "hostname"
 	TypeParameter            = "type"
 	NodeNameParameter        = "node"
+	ReserveSpaceParameter    = "reserveSpace"
 )
 
 // StorageClass Parameters are expected in the following schema:
@@ -21,6 +22,7 @@ parameters:
   type: nfs|hostpath
   shareProperties: rw=10.0.0.0/8,no_root_squash
   node: my-zfs-host
+  reserveSpace: true|false
 */
 
 type (
@@ -30,9 +32,10 @@ type (
 		// ParentDataset of the zpool. Needs to be existing on the target ZFS host.
 		ParentDataset string
 		// Hostname of the target ZFS host. Will be used to connect over SSH.
-		Hostname string
-		NFS      *NFSParameters
-		HostPath *HostPathParameters
+		Hostname     string
+		NFS          *NFSParameters
+		HostPath     *HostPathParameters
+		ReserveSpace bool
 	}
 	NFSParameters struct {
 		// ShareProperties specifies additional properties to pass to 'zfs create sharenfs=%s'.
@@ -57,9 +60,21 @@ func NewStorageClassParameters(parameters map[string]string) (*ZFSStorageClassPa
 	if strings.HasPrefix(parentDataset, "/") || strings.HasSuffix(parentDataset, "/") {
 		return nil, fmt.Errorf("%s must not begin or end with '/': %s", ParentDatasetParameter, parentDataset)
 	}
+
+	reserveSpaceValue, reserveSpaceValuePresent := parameters[ReserveSpaceParameter]
+	var reserveSpace bool
+	if !reserveSpaceValuePresent || strings.EqualFold(reserveSpaceValue, "true") {
+		reserveSpace = true
+	} else if strings.EqualFold(reserveSpaceValue, "false") {
+		reserveSpace = false
+	} else {
+		return nil, fmt.Errorf("invalid '%s' parameter value: %s", ReserveSpaceParameter, parameters[ReserveSpaceParameter])
+	}
+
 	p := &ZFSStorageClassParameters{
 		ParentDataset: parentDataset,
 		Hostname:      parameters[HostnameParameter],
+		ReserveSpace:  reserveSpace,
 	}
 	typeParam := parameters[TypeParameter]
 	switch typeParam {

pkg/provisioner/provision.go

@@ -41,10 +41,13 @@ func (p *ZFSProvisioner) Provision(ctx context.Context, options controller.Provi
 	storageRequest := options.PVC.Spec.Resources.Requests[v1.ResourceStorage]
 	storageRequestBytes := strconv.FormatInt(storageRequest.Value(), 10)
 	properties[RefQuotaProperty] = storageRequestBytes
-	properties[RefReservationProperty] = storageRequestBytes
 	properties[ManagedByProperty] = p.InstanceName
 	properties[ReclaimPolicyProperty] = string(reclaimPolicy)
 
+	if parameters.ReserveSpace {
+		properties[RefReservationProperty] = storageRequestBytes
+	}
+
 	dataset, err := p.zfs.CreateDataset(datasetPath, parameters.Hostname, properties)
 	if err != nil {
 		return nil, controller.ProvisioningFinished, fmt.Errorf("creating ZFS dataset failed: %w", err)

pkg/zfs/zfs.go

@@ -7,7 +7,7 @@ import (
 	"os/exec"
 	"sync"
 
-	gozfs "github.com/mistifyio/go-zfs"
+	gozfs "github.com/mistifyio/go-zfs/v3"
 	"k8s.io/klog/v2"
 )
 

test/provision_integration_test.go

@@ -6,6 +6,7 @@ import (
 	"bufio"
 	"context"
 	"flag"
+	gozfs "github.com/mistifyio/go-zfs/v3"
 	"math/rand"
 	"os"
 	"strconv"
@@ -30,13 +31,15 @@ var (
 
 type ProvisionTestSuit struct {
 	suite.Suite
-	p       *provisioner.ZFSProvisioner
-	dataset string
+	p               *provisioner.ZFSProvisioner
+	datasetPrefix   string
+	createdDatasets []string
 }
 
 func TestProvisionSuite(t *testing.T) {
 	s := ProvisionTestSuit{
-		dataset: "pv-test-" + strconv.Itoa(rand.Int()),
+		datasetPrefix:   "pv-test-" + strconv.Itoa(rand.Int()),
+		createdDatasets: make([]string, 0),
 	}
 	suite.Run(t, &s)
 }
@@ -52,35 +55,85 @@ func (suite *ProvisionTestSuit) SetupSuite() {
 }
 
 func (suite *ProvisionTestSuit) TearDownSuite() {
-	err := zfs.NewInterface().DestroyDataset(&zfs.Dataset{
-		Name:     *parentDataset + "/" + suite.dataset,
-		Hostname: "host",
-	}, zfs.DestroyRecursively)
-	require.NoError(suite.T(), err)
+	for _, dataset := range suite.createdDatasets {
+		err := zfs.NewInterface().DestroyDataset(&zfs.Dataset{
+			Name:     *parentDataset + "/" + dataset,
+			Hostname: "host",
+		}, zfs.DestroyRecursively)
+		require.NoError(suite.T(), err)
+	}
 }
 
-func (suite *ProvisionTestSuit) TestProvisionDataset() {
+func (suite *ProvisionTestSuit) TestDefaultProvisionDataset() {
+	dataset := provisionDataset(suite, "default", map[string]string{
+		provisioner.ParentDatasetParameter:   *parentDataset,
+		provisioner.HostnameParameter:        "localhost",
+		provisioner.TypeParameter:            "nfs",
+		provisioner.SharePropertiesParameter: "rw,no_root_squash",
+	})
+	assertZfsReservation(suite.T(), dataset, true)
+}
+
+func (suite *ProvisionTestSuit) TestThickProvisionDataset() {
+	dataset := provisionDataset(suite, "thick", map[string]string{
+		provisioner.ParentDatasetParameter:   *parentDataset,
+		provisioner.HostnameParameter:        "localhost",
+		provisioner.TypeParameter:            "nfs",
+		provisioner.SharePropertiesParameter: "rw,no_root_squash",
+		provisioner.ReserveSpaceParameter:    "true",
+	})
+	assertZfsReservation(suite.T(), dataset, true)
+}
+
+func (suite *ProvisionTestSuit) TestThinProvisionDataset() {
+	dataset := provisionDataset(suite, "thin", map[string]string{
+		provisioner.ParentDatasetParameter:   *parentDataset,
+		provisioner.HostnameParameter:        "localhost",
+		provisioner.TypeParameter:            "nfs",
+		provisioner.SharePropertiesParameter: "rw,no_root_squash",
+		provisioner.ReserveSpaceParameter:    "false",
+	})
+	assertZfsReservation(suite.T(), dataset, false)
+}
+
+func provisionDataset(suite *ProvisionTestSuit, name string, parameters map[string]string) string {
 	t := suite.T()
-	fullDataset := "/" + *parentDataset + "/" + suite.dataset
+	pvName := suite.datasetPrefix + "_" + name
+	fullDataset := *parentDataset + "/" + pvName
+	datasetDirectory := "/" + fullDataset
 	policy := v1.PersistentVolumeReclaimRetain
 	options := controller.ProvisionOptions{
-		PVName: suite.dataset,
+		PVName: pvName,
 		PVC:    newClaim(resource.MustParse("10M"), []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce, v1.ReadOnlyMany}),
 		StorageClass: &storagev1.StorageClass{
-			Parameters: map[string]string{
-				provisioner.ParentDatasetParameter:   *parentDataset,
-				provisioner.HostnameParameter:        "localhost",
-				provisioner.TypeParameter:            "nfs",
-				provisioner.SharePropertiesParameter: "rw,no_root_squash",
-			},
+			Parameters:    parameters,
 			ReclaimPolicy: &policy,
 		},
 	}
 
 	_, _, err := suite.p.Provision(context.Background(), options)
+	suite.createdDatasets = append(suite.createdDatasets, pvName)
 	assert.NoError(t, err)
-	require.DirExists(t, fullDataset)
-	assertNfsExport(t, fullDataset)
+	require.DirExists(t, datasetDirectory)
+	assertNfsExport(t, datasetDirectory)
+	return fullDataset
+}
+
+func assertZfsReservation(t *testing.T, datasetName string, reserve bool) {
+	dataset, err := gozfs.GetDataset(datasetName)
+	assert.NoError(t, err)
+
+	refreserved, err := dataset.GetProperty("refreservation")
+	assert.NoError(t, err)
+
+	refquota, err := dataset.GetProperty("refquota")
+	assert.NoError(t, err)
+
+	if reserve {
+		assert.Equal(t, refquota, refreserved)
+	} else {
+		assert.Equal(t, "none", refreserved)
+	}
 }
 
 func assertNfsExport(t *testing.T, fullDataset string) {