Test/OIDC s3 mount (#85)

* save state - not getting s3 csi to work
* wip testing oidc with s3
* it's working! First trrrryyyyy :D

Signed-off-by: vsoch <vsoch@users.noreply.github.com>

.dockerignore

@@ -1,4 +1,5 @@
 # More info: https://docs.docker.com/engine/reference/builder/#dockerignore-file
 # Ignore build and test binaries.
+examples/
 bin/
 testbin/

api/v1alpha1/minicluster_types.go

@@ -180,11 +180,31 @@ type MiniClusterVolume struct {
 	// +optional
 	Annotations map[string]string `json:"annotations"`
 
+	// Optional volume attributes
+	// +optional
+	Attributes map[string]string `json:"attributes"`
+
+	// Volume handle, falls back to storage class name
+	// if not defined
+	// +optional
+	VolumeHandle string `json:"volumeHandle"`
+
 	// +kubebuilder:default="hostpath"
 	// +default="hostpath"
 	// +optional
 	StorageClass string `json:"storageClass,omitempty"`
 
+	// Storage driver, e.g., gcs.csi.ofek.dev
+	// Only needed if not using hostpath
+	// +optional
+	Driver string `json:"driver"`
+
+	// Delete the persistent volume on cleanup
+	// +kubebuilder:default=true
+	// +default=true
+	// +optional
+	Delete bool `json:"delete,omitempty"`
+
 	// Secret reference in Kubernetes with service account role
 	// +optional
 	Secret string `json:"secret"`

api/v1alpha1/swagger.json

@@ -436,11 +436,29 @@
             "default": ""
           }
         },
+        "attributes": {
+          "description": "Optional volume attributes",
+          "type": "object",
+          "additionalProperties": {
+            "type": "string",
+            "default": ""
+          }
+        },
         "capacity": {
           "description": "Capacity (string) for PVC (storage request) to create PV",
           "type": "string",
           "default": "5Gi"
         },
+        "delete": {
+          "description": "Delete the persistent volume on cleanup",
+          "type": "boolean",
+          "default": true
+        },
+        "driver": {
+          "description": "Storage driver, e.g., gcs.csi.ofek.dev Only needed if not using hostpath",
+          "type": "string",
+          "default": ""
+        },
         "labels": {
           "type": "object",
           "additionalProperties": {
@@ -465,6 +483,11 @@
         "storageClass": {
           "type": "string",
           "default": "hostpath"
+        },
+        "volumeHandle": {
+          "description": "Volume handle, falls back to storage class name if not defined",
+          "type": "string",
+          "default": ""
         }
       }
     },

config/crd/bases/flux-framework.org_miniclusters.yaml

@@ -294,11 +294,24 @@ spec:
                         type: string
                       description: Annotations for persistent volume claim
                       type: object
+                    attributes:
+                      additionalProperties:
+                        type: string
+                      description: Optional volume attributes
+                      type: object
                     capacity:
                       default: 5Gi
                       description: Capacity (string) for PVC (storage request) to
                         create PV
                       type: string
+                    delete:
+                      default: true
+                      description: Delete the persistent volume on cleanup
+                      type: boolean
+                    driver:
+                      description: Storage driver, e.g., gcs.csi.ofek.dev Only needed
+                        if not using hostpath
+                      type: string
                     labels:
                       additionalProperties:
                         type: string
@@ -316,6 +329,10 @@ spec:
                     storageClass:
                       default: hostpath
                       type: string
+                    volumeHandle:
+                      description: Volume handle, falls back to storage class name
+                        if not defined
+                      type: string
                   required:
                   - path
                   type: object

controllers/flux/minicluster.go

@@ -139,20 +139,24 @@ func (r *MiniClusterReconciler) cleanupPodsStorage(
 	// The job deletion should handle pods, next delete pvc and pv per each volume
 	// Must be deleted in that order, per internet advice :)
 	for volumeName := range cluster.Spec.Volumes {
+		volumeSpec := cluster.Spec.Volumes[volumeName]
 
 		claimName := fmt.Sprintf("%s-claim", volumeName)
 
-		// Only delete if we retrieve without error
+		// Only delete if we retrieve without error and user has requested
 		claim, err := r.getExistingPersistentVolumeClaim(ctx, cluster, claimName)
 		if err != nil {
 			r.log.Info("Volume Claim", "Deletion", claim.Name)
 			r.Client.Delete(ctx, claim)
 		}
 
-		pv, err := r.getExistingPersistentVolume(ctx, cluster, volumeName)
-		if err != nil {
-			r.log.Info("Volume", "Deletion", pv.Name)
-			r.Client.Delete(ctx, pv)
+		// Different request to delete
+		if volumeSpec.Delete {
+			pv, err := r.getExistingPersistentVolume(ctx, cluster, volumeName)
+			if err != nil {
+				r.log.Info("Volume", "Deletion", pv.Name)
+				r.Client.Delete(ctx, pv)
+			}
 		}
 	}
 	return ctrl.Result{Requeue: false}, nil

controllers/flux/volumes.go

@@ -167,18 +167,34 @@ func (r *MiniClusterReconciler) createPersistentVolume(
 		}
 
 	} else {
+
+		// VolumeHandle defaults to storage class name
+		// unless it is explicitly different!
+		volumeHandle := volume.StorageClass
+		if volume.VolumeHandle != "" {
+			volumeHandle = volume.VolumeHandle
+		}
 		pvsource = corev1.PersistentVolumeSource{
 			CSI: &corev1.CSIPersistentVolumeSource{
 
 				// Choose for the user for now.
-				Driver: "gcs.csi.ofek.dev",
+				Driver: volume.Driver,
 
-				// Name in storageclass metadata
-				VolumeHandle: "csi-gcs",
+				// Name in storageclass metadata, also what we use for name
+				VolumeHandle: volumeHandle,
 				NodePublishSecretRef: &corev1.SecretReference{
 					Namespace: volume.SecretNamespace,
 					Name:      volume.Secret,
 				},
+				ControllerPublishSecretRef: &corev1.SecretReference{
+					Namespace: volume.SecretNamespace,
+					Name:      volume.Secret,
+				},
+				NodeStageSecretRef: &corev1.SecretReference{
+					Namespace: volume.SecretNamespace,
+					Name:      volume.Secret,
+				},
+				VolumeAttributes: volume.Attributes,
 			},
 		}
 	}