Add support for volume populators (#2482)

* Add support for volume populators in CDI

This commit enables the use of volume populators in CDI, so datavolume-owned PVCs can be populated using custom logic.

Volume populators are CRDs used to populate volumes externally, independently of CDI. These CRDs can now be specified using the new DataSourceRef API field in the DataVolume spec.

When a DataVolume is created with a populated DataSourceRef field, the datavolume-controller creates the corresponding PVC accordingly but skips all the population-related steps. Once the PVC is bound, the DV phase changes to succeeded.

Signed-off-by: Alvaro Romero <alromero@redhat.com>

* Modify CDI test infrastructure to support testing of external populators

This commit introduces several changes to CDI ci to support the testing of DataVolumes with external populators:
* A sample volume populator is now deployed in the test infrastructure, in a similar way as bad-webserver or test-proxy. This populator will be used in functional tests from now on.
* A new test file with external population tests has been introduced in the tests directory

Signed-off-by: Alvaro Romero <alromero@redhat.com>

* Update dependencies to include lib-volume-populator library

Signed-off-by: Alvaro Romero <alromero@redhat.com>

* Add functional tests for proper coverage of external population of DataVolumes

Signed-off-by: Alvaro Romero <alromero@redhat.com>

* Minor fixes on external-population logic for DataVolumes:
* Added comments for exported structs
* Removed non-inclusive language
* Improved error messages in webhooks
* Fixed logic on datavolume-controller

Signed-off-by: Alvaro Romero <alromero@redhat.com>

* Improve DataVolume external-population logic when using the old 'DataSource' API

This commit introduces several changes into the datavolume external-population controller to improve its behavior when using the DataSource field.

It also introduces minor fixes on the generic populator logic.

Signed-off-by: Alvaro Romero <alromero@redhat.com>

* Add unit tests for external-population controller and DV admission

Signed-off-by: Alvaro Romero <alromero@redhat.com>

Signed-off-by: Alvaro Romero <alromero@redhat.com>

BUILD.bazel

@@ -53,6 +53,7 @@ container_bundle(
     images = {
         "$(container_prefix)/cdi-func-test-bad-webserver:$(container_tag)": "//tools/cdi-func-test-bad-webserver:cdi-func-test-bad-webserver-image",
         "$(container_prefix)/cdi-func-test-proxy:$(container_tag)": "//tools/cdi-func-test-proxy:cdi-func-test-proxy-image",
+        "$(container_prefix)/cdi-func-test-sample-populator:$(container_tag)": "//tools/cdi-func-test-sample-populator:cdi-func-test-sample-populator-image",
         "$(container_prefix)/cdi-func-test-file-host-init:$(container_tag)": "//tools/cdi-func-test-file-host-init:cdi-func-test-file-host-init-image",
         "$(container_prefix)/cdi-func-test-file-host-http:$(container_tag)": "//tools/cdi-func-test-file-host-init:cdi-func-test-file-host-http-image",
         "$(container_prefix)/cdi-func-test-registry-init:$(container_tag)": "//tools/cdi-func-test-registry-init:cdi-func-test-registry-init-image",
@@ -72,6 +73,7 @@ container_bundle(
     images = {
         "$(container_prefix)/cdi-func-test-bad-webserver:$(container_tag)": "//tools/cdi-func-test-bad-webserver:cdi-func-test-bad-webserver-image",
         "$(container_prefix)/cdi-func-test-proxy:$(container_tag)": "//tools/cdi-func-test-proxy:cdi-func-test-proxy-image",
+        "$(container_prefix)/cdi-func-test-sample-populator:$(container_tag)": "//tools/cdi-func-test-sample-populator:cdi-func-test-sample-populator-image",
         "$(container_prefix)/cdi-func-test-file-host-init:$(container_tag)": "//tools/cdi-func-test-file-host-init:cdi-func-test-file-host-init-image",
         "$(container_prefix)/cdi-func-test-file-host-http:$(container_tag)": "//tools/cdi-func-test-file-host-init:cdi-func-test-file-host-http-image",
         "$(container_prefix)/cdi-func-test-registry-init:$(container_tag)": "//tools/cdi-func-test-registry-init:cdi-func-test-registry-init-image",

api/openapi-spec/swagger.json

@@ -5855,7 +5855,11 @@
       }
      },
      "dataSource": {
-      "description": "This field can be used to specify either: * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) * An existing PVC (PersistentVolumeClaim) * An existing custom resource that implements data population (Alpha) In order to use custom resource types that implement data population, the AnyVolumeDataSource feature gate must be enabled. If the provisioner or an external controller can support the specified data source, it will create a new volume based on the contents of the specified data source.",
+      "description": "This field can be used to specify either: * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) * An existing PVC (PersistentVolumeClaim) * An existing custom resource that implements data population (Alpha) In order to use custom resource types that implement data population, the AnyVolumeDataSource feature gate must be enabled. If the provisioner or an external controller can support the specified data source, it will create a new volume based on the contents of the specified data source. If the AnyVolumeDataSource feature gate is enabled, this field will always have the same contents as the DataSourceRef field.",
+      "$ref": "#/definitions/v1.TypedLocalObjectReference"
+     },
+     "dataSourceRef": {
+      "description": "Specifies the object from which to populate the volume with data, if a non-empty volume is desired. This may be any local object from a non-empty API group (non core object) or a PersistentVolumeClaim object. When this field is specified, volume binding will only succeed if the type of the specified object matches some installed volume populator or dynamic provisioner. This field will replace the functionality of the DataSource field and as such if both fields are non-empty, they must have the same value. For backwards compatibility, both fields (DataSource and DataSourceRef) will be set to the same value automatically if one of them is empty and the other is non-empty. There are two important differences between DataSource and DataSourceRef: * While DataSource only allows two specific types of objects, DataSourceRef allows any non-core object, as well as PersistentVolumeClaim objects. * While DataSource ignores disallowed values (dropping them), DataSourceRef preserves all values, and generates an error if a disallowed value is specified. (Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled.",
       "$ref": "#/definitions/v1.TypedLocalObjectReference"
      },
      "resources": {

cluster-sync/sync-os-ci.sh

@@ -96,6 +96,7 @@ _kubectl apply -f "./_out/manifests/bad-webserver.yaml"
 _kubectl apply -f "./_out/manifests/file-host.yaml"
 _kubectl apply -f "./_out/manifests/registry-host.yaml"
 _kubectl apply -f "./_out/manifests/test-proxy.yaml"
+_kubectl apply -f "./_out/manifests/sample-populator.yaml"
 # Imageio test service:
 _kubectl apply -f "./_out/manifests/imageio.yaml"
 # vCenter (VDDK) test service:

cluster-sync/sync.sh

@@ -172,6 +172,7 @@ if [ "${CDI_SYNC}" == "test-infra" ]; then
   _kubectl apply -f "./_out/manifests/file-host.yaml"
   _kubectl apply -f "./_out/manifests/registry-host.yaml"
   _kubectl apply -f "./_out/manifests/test-proxy.yaml"
+  _kubectl apply -f "./_out/manifests/sample-populator.yaml"
   _kubectl apply -f "./_out/manifests/uploadproxy-nodeport.yaml"
   # Imageio test service:
   _kubectl apply -f "./_out/manifests/imageio.yaml"

cmd/cdi-controller/controller.go

@@ -218,6 +218,10 @@ func start(ctx context.Context, cfg *rest.Config) {
 		klog.Errorf("Unable to setup datavolume clone controller: %v", err)
 		os.Exit(1)
 	}
+	if _, err := dvc.NewPopulatorController(ctx, mgr, log, installerLabels); err != nil {
+		klog.Errorf("Unable to setup datavolume external-population controller: %v", err)
+		os.Exit(1)
+	}
 
 	if _, err := controller.NewImportController(mgr, log, importerImage, pullPolicy, verbose, installerLabels); err != nil {
 		klog.Errorf("Unable to setup import controller: %v", err)

go.mod

@@ -19,6 +19,7 @@ require (
 	github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/kubernetes-csi/external-snapshotter/client/v6 v6.0.1
+	github.com/kubernetes-csi/lib-volume-populator v1.2.0
 	github.com/onsi/ginkgo v1.16.5
 	github.com/onsi/gomega v1.19.0
 	github.com/openshift/api v0.0.0
@@ -30,26 +31,26 @@ require (
 	github.com/ovirt/go-ovirt-client v0.9.0
 	github.com/ovirt/go-ovirt-client-log-klog v1.0.0
 	github.com/pkg/errors v0.9.1
-	github.com/prometheus/client_golang v1.11.1
+	github.com/prometheus/client_golang v1.12.1
 	github.com/prometheus/client_model v0.2.0
 	github.com/rs/cors v1.7.0
 	github.com/ulikunitz/xz v0.5.10
 	github.com/vmware/govmomi v0.23.1
 	go.uber.org/zap v1.19.1
-	golang.org/x/sys v0.0.0-20220209214540-3681064d5158
+	golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f
 	gopkg.in/fsnotify.v1 v1.4.7
 	gopkg.in/square/go-jose.v2 v2.5.1
-	k8s.io/api v0.23.5
+	k8s.io/api v0.25.0
 	k8s.io/apiextensions-apiserver v0.23.5
-	k8s.io/apimachinery v0.23.5
+	k8s.io/apimachinery v0.25.0
 	k8s.io/apiserver v0.23.0
 	k8s.io/client-go v12.0.0+incompatible
 	k8s.io/cluster-bootstrap v0.0.0
 	k8s.io/code-generator v0.23.3
-	k8s.io/klog/v2 v2.40.1
+	k8s.io/klog/v2 v2.70.1
 	k8s.io/kube-aggregator v0.23.0
-	k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf
-	k8s.io/utils v0.0.0-20211116205334-6203023598ed
+	k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1
+	k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed
 	kubevirt.io/containerized-data-importer-api v0.0.0
 	kubevirt.io/controller-lifecycle-operator-sdk v0.2.4
 	kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90
@@ -109,8 +110,8 @@ require (
 	github.com/opencontainers/runc v1.1.2 // indirect
 	github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 // indirect
 	github.com/ovirt/go-ovirt-client-log/v2 v2.2.0 // indirect
-	github.com/prometheus/common v0.28.0 // indirect
-	github.com/prometheus/procfs v0.6.0 // indirect
+	github.com/prometheus/common v0.32.1 // indirect
+	github.com/prometheus/procfs v0.7.3 // indirect
 	github.com/robfig/cron v1.2.0 // indirect
 	github.com/sirupsen/logrus v1.8.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
@@ -121,25 +122,25 @@ require (
 	go.uber.org/multierr v1.6.0 // indirect
 	golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29 // indirect
 	golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect
-	golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect
+	golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect
 	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
 	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
+	golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
 	golang.org/x/tools v0.1.10 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/protobuf v1.27.1 // indirect
+	google.golang.org/protobuf v1.28.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0 // indirect
-	k8s.io/component-base v0.23.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/component-base v0.25.0 // indirect
 	k8s.io/gengo v0.0.0-20211129171323-c02415ce4185 // indirect
-	sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect
+	sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
 	sigs.k8s.io/kube-storage-version-migrator v0.0.4 // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )