Skip to content

Commit

Permalink
add integration tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@jmdeal
jmdeal committed Dec 5, 2023
1 parent dafe148 commit 2f18228
Showing 1 changed file with 131 additions and 81 deletions.
212 changes: 131 additions & 81 deletions test/suites/integration/storage_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,105 +15,155 @@ limitations under the License.
package integration_test

import (
"context"
"fmt"

"github.com/aws/aws-sdk-go/aws"
appsv1 "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
storagev1 "k8s.io/api/storage/v1"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"

"github.com/aws/aws-sdk-go/aws"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/samber/lo"

"github.com/aws/karpenter/pkg/errors"

"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/karpenter/pkg/test"
)

// This test requires the EBS CSI driver to be installed
var _ = Describe("Dynamic PVC", func() {
It("should run a pod with a dynamic persistent volume", func() {
// Ensure that the EBS driver is installed, or we can't run the test.
var ds appsv1.DaemonSet
if err := env.Client.Get(env.Context, client.ObjectKey{
Namespace: "kube-system",
Name: "ebs-csi-node",
}, &ds); err != nil {
if errors.IsNotFound(err) {
Skip(fmt.Sprintf("skipping dynamic PVC test due to missing EBS driver %s", err))
} else {
Fail(fmt.Sprintf("determining EBS driver status, %s", err))
}
}
storageClassName := "ebs-sc-test"
bindMode := storagev1.VolumeBindingWaitForFirstConsumer
sc := test.StorageClass(test.StorageClassOptions{
ObjectMeta: metav1.ObjectMeta{
Name: storageClassName,
},
Provisioner: aws.String("ebs.csi.aws.com"),
VolumeBindingMode: &bindMode,
const IsDefaultStorageClassAnnotation = "storageclass.kubernetes.io/is-default-class"

var _ = Describe("Persistent Volumes", func() {
Context("Static", func() {
It("should run a pod with a pre-bound persistent volume (empty storage class)", func() {
pvc := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{
VolumeName: "test-volume",
StorageClassName: lo.ToPtr(""),
})
pv := test.PersistentVolume(test.PersistentVolumeOptions{
ObjectMeta: metav1.ObjectMeta{
Name: pvc.Spec.VolumeName,
},
})
pod := test.Pod(test.PodOptions{
PersistentVolumeClaims: []string{pvc.Name},
})

env.ExpectCreated(nodeClass, nodePool, pv, pvc, pod)
env.EventuallyExpectHealthy(pod)
env.ExpectCreatedNodeCount("==", 1)
})

pvc := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{
ObjectMeta: metav1.ObjectMeta{
Name: "ebs-claim",
},
StorageClassName: aws.String(storageClassName),
Resources: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceStorage: resource.MustParse("5Gi")}},
It("should run a pod with a pre-bounnd persistent volume (empty storage class + no default)", func() {
restoreDefaults := ExpectDefaultStorageClassesRemoved(env.Context, env.Client)
defer restoreDefaults()

pvc := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{
VolumeName: "test-volume",
StorageClassName: lo.ToPtr(""),
})
pv := test.PersistentVolume(test.PersistentVolumeOptions{
ObjectMeta: metav1.ObjectMeta{
Name: pvc.Spec.VolumeName,
},
})
pod := test.Pod(test.PodOptions{
PersistentVolumeClaims: []string{pvc.Name},
})

env.ExpectCreated(nodeClass, nodePool, pv, pvc, pod)
env.EventuallyExpectHealthy(pod)
env.ExpectCreatedNodeCount("==", 1)
})

pod := test.Pod(test.PodOptions{
PersistentVolumeClaims: []string{pvc.Name},
It("should run a pod with a pre-bound persistent volume (non-existent storage class)", func() {
pvc := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{
VolumeName: "test-volume",
StorageClassName: lo.ToPtr("non-existent-storage-class"),
})
pv := test.PersistentVolume(test.PersistentVolumeOptions{
ObjectMeta: metav1.ObjectMeta{
Name: pvc.Spec.VolumeName,
},
StorageClassName: "non-existent-storage-class",
})
pod := test.Pod(test.PodOptions{
PersistentVolumeClaims: []string{pvc.Name},
})
env.ExpectCreated(nodeClass, nodePool, pv, pvc, pod)
env.EventuallyExpectHealthy(pod)
env.ExpectCreatedNodeCount("==", 1)
})

env.ExpectCreated(nodeClass, nodePool, sc, pvc, pod)
env.EventuallyExpectHealthy(pod)
env.ExpectCreatedNodeCount("==", 1)
env.ExpectDeleted(pod)
})
})

var _ = Describe("Static PVC", func() {
It("should run a pod with a static persistent volume", func() {
storageClassName := "nfs-test"
bindMode := storagev1.VolumeBindingWaitForFirstConsumer
sc := test.StorageClass(test.StorageClassOptions{
ObjectMeta: metav1.ObjectMeta{
Name: storageClassName,
},
VolumeBindingMode: &bindMode,
})

pv := test.PersistentVolume(test.PersistentVolumeOptions{
ObjectMeta: metav1.ObjectMeta{Name: "nfs-test-volume"},
StorageClassName: "nfs-test",
Context("Dynamic", func() {
var storageClass *storagev1.StorageClass
BeforeEach(func() {
// Ensure that the EBS driver is installed, or we can't run the test.
var ds appsv1.DaemonSet
if err := env.Client.Get(env.Context, client.ObjectKey{
Namespace: "kube-system",
Name: "ebs-csi-node",
}, &ds); err != nil {
if errors.IsNotFound(err) {
Skip(fmt.Sprintf("skipping dynamic PVC test due to missing EBS driver %s", err))
} else {
Fail(fmt.Sprintf("determining EBS driver status, %s", err))
}
}
storageClass = test.StorageClass(test.StorageClassOptions{
ObjectMeta: metav1.ObjectMeta{
Name: "test-storage-class",
},
Provisioner: aws.String("ebs.csi.aws.com"),
VolumeBindingMode: lo.ToPtr(storagev1.VolumeBindingWaitForFirstConsumer),
})
})

// the server here doesn't need to actually exist for the pod to start running
pv.Spec.NFS = &v1.NFSVolumeSource{
Server: "fake.server",
Path: "/some/path",
}
pv.Spec.CSI = nil

pvc := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{
ObjectMeta: metav1.ObjectMeta{
Name: "nfs-claim",
},
StorageClassName: aws.String(storageClassName),
VolumeName: pv.Name,
Resources: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceStorage: resource.MustParse("5Gi")}},
It("should run a pod with a dynamic persistent volume (non-existent storage class)", func() {
pvc := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{
StorageClassName: &storageClass.Name,
})
pod := test.Pod(test.PodOptions{
PersistentVolumeClaims: []string{pvc.Name},
})

env.ExpectCreated(nodeClass, nodePool, storageClass, pvc, pod)
env.EventuallyExpectHealthy(pod)
env.ExpectCreatedNodeCount("==", 1)
})

pod := test.Pod(test.PodOptions{
PersistentVolumeClaims: []string{pvc.Name},
It("should run a pod with a generic ephemeral volume", func() {
pod := test.Pod(test.PodOptions{
EphemeralVolumeTemplates: []test.EphemeralVolumeTemplateOptions{{
StorageClassName: &storageClass.Name,
}},
})

env.ExpectCreated(nodeClass, nodePool, storageClass, pod)
env.EventuallyExpectHealthy(pod)
env.ExpectCreatedNodeCount("==", 1)
})

env.ExpectCreated(nodeClass, nodePool, sc, pv, pvc, pod)
env.EventuallyExpectHealthy(pod)
env.ExpectCreatedNodeCount("==", 1)
env.ExpectDeleted(pod)
})
})

func ExpectDefaultStorageClassesRemoved(ctx context.Context, kubeClient client.Client) func() {
classes := &storagev1.StorageClassList{}
Expect(kubeClient.List(ctx, classes)).To(Succeed())
defaultClasses := lo.Map(lo.Filter(classes.Items, func(sc storagev1.StorageClass, _ int) bool {
return sc.Annotations[IsDefaultStorageClassAnnotation] == "true"
}), func(sc storagev1.StorageClass, _ int) *storagev1.StorageClass {
return lo.ToPtr(sc)
})
for _, sc := range defaultClasses {
persisted := sc.DeepCopy()
delete(sc.Annotations, IsDefaultStorageClassAnnotation)
Expect(kubeClient.Patch(ctx, sc, client.MergeFrom(persisted))).To(Succeed())
}

return func() {
for _, sc := range defaultClasses {
persisted := sc.DeepCopy()
sc.Annotations[IsDefaultStorageClassAnnotation] = "true"
Expect(kubeClient.Patch(ctx, sc, client.MergeFrom(persisted))).To(Succeed())
}
}
}

0 comments on commit 2f18228

Please sign in to comment.