test: v1beta1 e2e testing (#4873)

Co-authored-by: njtran <njtran@amazon.com>

.github/workflows/e2e-matrix.yaml

@@ -49,7 +49,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        suite: [Integration, Machine, Consolidation, Utilization, Interruption, Drift, Expiration, Chaos, IPv6]
+        suite: [Beta/Drift, Alpha/Integration, Alpha/Machine, Alpha/Consolidation, Alpha/Utilization, Alpha/Interruption, Alpha/Drift, Alpha/Expiration, Alpha/Chaos, Alpha/IPv6]
     uses: ./.github/workflows/e2e.yaml
     with:
       suite: ${{ matrix.suite }}

.github/workflows/e2e.yaml

@@ -15,16 +15,17 @@ on:
         type: choice
         required: true
         options:
-          - Integration
-          - Machine
-          - Consolidation
-          - Utilization
-          - Interruption
-          - Drift
-          - Expiration
-          - Chaos
-          - IPv6
-          - Scale
+          - Beta/Drift
+          - Alpha/Integration
+          - Alpha/Machine
+          - Alpha/Consolidation
+          - Alpha/Utilization
+          - Alpha/Interruption
+          - Alpha/Drift
+          - Alpha/Expiration
+          - Alpha/Chaos
+          - Alpha/IPv6
+          - Alpha/Scale
       k8s_version:
         type: choice
         options:
@@ -107,7 +108,7 @@ jobs:
           cluster_name: ${{ env.CLUSTER_NAME }}
           k8s_version: ${{ inputs.k8s_version }}
           eksctl_version: ${{ inputs.eksctl_version }}
-          ip_family: ${{ inputs.suite == 'IPv6' && 'IPv6' || 'IPv4' }} # Set the value to IPv6 if IPv6 suite, else IPv4
+          ip_family: ${{ contains(inputs.suite, 'IPv6') && 'IPv6' || 'IPv4' }} # Set the value to IPv6 if IPv6 suite, else IPv4
           git_ref: ${{ inputs.git_ref }}
       - name: install prometheus
         uses: ./.github/actions/e2e/install-prometheus

test/pkg/debug/monitor.go

@@ -83,6 +83,7 @@ func (m *Monitor) Stop() {
 func newControllers(kubeClient client.Client) []controller.Controller {
 	return []controller.Controller{
 		NewMachineController(kubeClient),
+		NewNodeClaimController(kubeClient),
 		NewNodeController(kubeClient),
 		NewPodController(kubeClient),
 	}

test/pkg/debug/nodeclaim.go

@@ -0,0 +1,82 @@
+/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package debug
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"k8s.io/apimachinery/pkg/api/errors"
+	controllerruntime "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+
+	corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1"
+	corecontroller "github.com/aws/karpenter-core/pkg/operator/controller"
+)
+
+type NodeClaimController struct {
+	kubeClient client.Client
+}
+
+func NewNodeClaimController(kubeClient client.Client) *NodeClaimController {
+	return &NodeClaimController{
+		kubeClient: kubeClient,
+	}
+}
+
+func (c *NodeClaimController) Name() string {
+	return "nodeclaim"
+}
+
+func (c *NodeClaimController) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) {
+	nc := &corev1beta1.NodeClaim{}
+	if err := c.kubeClient.Get(ctx, req.NamespacedName, nc); err != nil {
+		if errors.IsNotFound(err) {
+			fmt.Printf("[DELETED %s] NODECLAIM %s\n", time.Now().Format(time.RFC3339), req.NamespacedName.String())
+		}
+		return reconcile.Result{}, client.IgnoreNotFound(err)
+	}
+	fmt.Printf("[CREATED/UPDATED %s] NODECLAIM %s %s\n", time.Now().Format(time.RFC3339), req.NamespacedName.Name, c.GetInfo(nc))
+	return reconcile.Result{}, nil
+}
+
+func (c *NodeClaimController) GetInfo(nc *corev1beta1.NodeClaim) string {
+	return fmt.Sprintf("ready=%t launched=%t registered=%t initialized=%t",
+		nc.StatusConditions().IsHappy(),
+		nc.StatusConditions().GetCondition(corev1beta1.Launched).IsTrue(),
+		nc.StatusConditions().GetCondition(corev1beta1.Registered).IsTrue(),
+		nc.StatusConditions().GetCondition(corev1beta1.Initialized).IsTrue(),
+	)
+}
+
+func (c *NodeClaimController) Builder(_ context.Context, m manager.Manager) corecontroller.Builder {
+	return corecontroller.Adapt(controllerruntime.
+		NewControllerManagedBy(m).
+		For(&corev1beta1.NodeClaim{}).
+		WithEventFilter(predicate.Funcs{
+			UpdateFunc: func(e event.UpdateEvent) bool {
+				oldNodeClaim := e.ObjectOld.(*corev1beta1.NodeClaim)
+				newNodeClaim := e.ObjectNew.(*corev1beta1.NodeClaim)
+				return c.GetInfo(oldNodeClaim) != c.GetInfo(newNodeClaim)
+			},
+		}).
+		WithOptions(controller.Options{MaxConcurrentReconciles: 10}))
+}

test/pkg/environment/aws/setup.go

@@ -20,6 +20,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/aws/karpenter/pkg/apis/v1alpha1"
+	"github.com/aws/karpenter/pkg/apis/v1beta1"
 )
 
 var persistedSettings []v1.EnvVar
@@ -28,6 +29,7 @@ var persistedSettingsLegacy = &v1.ConfigMap{}
 var (
 	CleanableObjects = []client.Object{
 		&v1alpha1.AWSNodeTemplate{},
+		&v1beta1.EC2NodeClass{},
 	}
 )
 
@@ -38,8 +40,8 @@ func (env *Environment) BeforeEach() {
 }
 
 func (env *Environment) Cleanup() {
-	env.Environment.CleanupObjects(CleanableObjects...)
 	env.Environment.Cleanup()
+	env.Environment.CleanupObjects(CleanableObjects...)
 }
 
 func (env *Environment) AfterEach() {

test/pkg/environment/common/expectations.go

@@ -40,6 +40,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/aws/karpenter-core/pkg/apis/v1alpha5"
+	corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1"
 	pscheduling "github.com/aws/karpenter-core/pkg/controllers/provisioning/scheduling"
 	"github.com/aws/karpenter-core/pkg/scheduling"
 	"github.com/aws/karpenter-core/pkg/test"
@@ -583,6 +584,29 @@ func (env *Environment) EventuallyExpectMachinesReady(machines ...*v1alpha5.Mach
 	}).Should(Succeed())
 }
 
+func (env *Environment) EventuallyExpectCreatedNodeClaimCount(comparator string, count int) []*corev1beta1.NodeClaim {
+	GinkgoHelper()
+	By(fmt.Sprintf("waiting for created nodeclaims to be %s to %d", comparator, count))
+	nodeClaimList := &corev1beta1.NodeClaimList{}
+	Eventually(func(g Gomega) {
+		g.Expect(env.Client.List(env.Context, nodeClaimList)).To(Succeed())
+		g.Expect(len(nodeClaimList.Items)).To(BeNumerically(comparator, count))
+	}).Should(Succeed())
+	return lo.Map(nodeClaimList.Items, func(nc corev1beta1.NodeClaim, _ int) *corev1beta1.NodeClaim {
+		return &nc
+	})
+}
+
+func (env *Environment) EventuallyExpectNodeClaimsReady(nodeClaims ...*corev1beta1.NodeClaim) {
+	Eventually(func(g Gomega) {
+		for _, nc := range nodeClaims {
+			temp := &corev1beta1.NodeClaim{}
+			g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(nc), temp)).Should(Succeed())
+			g.Expect(temp.StatusConditions().IsHappy()).To(BeTrue())
+		}
+	}).Should(Succeed())
+}
+
 func (env *Environment) GetNode(nodeName string) v1.Node {
 	GinkgoHelper()
 	var node v1.Node

test/pkg/environment/common/setup.go

@@ -34,9 +34,12 @@ import (
 
 	"github.com/aws/karpenter-core/pkg/apis"
 	"github.com/aws/karpenter-core/pkg/apis/v1alpha5"
+	corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1"
 	"github.com/aws/karpenter-core/pkg/operator/injection"
 	"github.com/aws/karpenter-core/pkg/test"
 	"github.com/aws/karpenter-core/pkg/utils/pod"
+	"github.com/aws/karpenter/pkg/apis/v1alpha1"
+	"github.com/aws/karpenter/pkg/apis/v1beta1"
 	"github.com/aws/karpenter/test/pkg/debug"
 )
 
@@ -50,10 +53,12 @@ var (
 		&v1.PersistentVolume{},
 		&storagev1.StorageClass{},
 		&v1alpha5.Provisioner{},
+		&corev1beta1.NodePool{},
 		&v1.LimitRange{},
 		&schedulingv1.PriorityClass{},
 		&v1.Node{},
 		&v1alpha5.Machine{},
+		&corev1beta1.NodeClaim{},
 	}
 )
 
@@ -65,9 +70,6 @@ func (env *Environment) BeforeEach() {
 	// Expect this cluster to be clean for test runs to execute successfully
 	env.ExpectCleanCluster()
 
-	var provisioners v1alpha5.ProvisionerList
-	Expect(env.Client.List(env.Context, &provisioners)).To(Succeed())
-	Expect(provisioners.Items).To(HaveLen(0), "expected no provisioners to exist")
 	env.Monitor.Reset()
 	env.StartingNodeCount = env.Monitor.NodeCountAtReset()
 }
@@ -88,6 +90,13 @@ func (env *Environment) ExpectCleanCluster() {
 		Expect(pods.Items[i].Namespace).ToNot(Equal("default"),
 			fmt.Sprintf("expected no pods in the `default` namespace, found %s/%s", pods.Items[i].Namespace, pods.Items[i].Name))
 	}
+	for _, obj := range []client.Object{&v1alpha5.Provisioner{}, &v1alpha1.AWSNodeTemplate{}, &corev1beta1.NodePool{}, &v1beta1.EC2NodeClass{}} {
+		metaList := &metav1.PartialObjectMetadataList{}
+		gvk := lo.Must(apiutil.GVKForObject(obj, env.Client.Scheme()))
+		metaList.SetGroupVersionKind(gvk)
+		Expect(env.Client.List(env.Context, metaList, client.Limit(1))).To(Succeed())
+		Expect(metaList.Items).To(HaveLen(0), fmt.Sprintf("expected no %s to exist", gvk.Kind))
+	}
 }
 
 func (env *Environment) Cleanup() {

test/suites/chaos/suite_test.go → test/suites/alpha/chaos/suite_test.go

@@ -54,7 +54,7 @@ func TestChaos(t *testing.T) {
 	AfterSuite(func() {
 		env.Stop()
 	})
-	RunSpecs(t, "Chaos")
+	RunSpecs(t, "Alpha/Chaos")
 }
 
 var _ = BeforeEach(func() { env.BeforeEach() })

test/suites/consolidation/suite_test.go → test/suites/alpha/consolidation/suite_test.go

@@ -49,7 +49,7 @@ func TestConsolidation(t *testing.T) {
 	AfterSuite(func() {
 		env.Stop()
 	})
-	RunSpecs(t, "Consolidation")
+	RunSpecs(t, "Alpha/Consolidation")
 }
 
 var _ = BeforeEach(func() { env.BeforeEach() })

test/suites/drift/suite_test.go → test/suites/alpha/drift/suite_test.go

@@ -55,7 +55,7 @@ func TestDrift(t *testing.T) {
 	AfterSuite(func() {
 		env.Stop()
 	})
-	RunSpecs(t, "Drift")
+	RunSpecs(t, "Alpha/Drift")
 }
 
 var _ = BeforeEach(func() {

test/suites/expiration/expiration_test.go → test/suites/alpha/expiration/expiration_test.go

@@ -51,7 +51,7 @@ func TestExpiration(t *testing.T) {
 	AfterSuite(func() {
 		env.Stop()
 	})
-	RunSpecs(t, "Expiration")
+	RunSpecs(t, "Alpha/Expiration")
 }
 
 var _ = BeforeEach(func() {

test/suites/integration/suite_test.go → test/suites/alpha/integration/suite_test.go

@@ -33,7 +33,7 @@ func TestIntegration(t *testing.T) {
 	AfterSuite(func() {
 		env.Stop()
 	})
-	RunSpecs(t, "Integration")
+	RunSpecs(t, "Alpha/Integration")
 }
 
 var _ = BeforeEach(func() { env.BeforeEach() })

test/suites/interruption/suite_test.go → test/suites/alpha/interruption/suite_test.go

@@ -50,7 +50,7 @@ func TestInterruption(t *testing.T) {
 	AfterSuite(func() {
 		env.Stop()
 	})
-	RunSpecs(t, "Interruption")
+	RunSpecs(t, "Alpha/Interruption")
 }
 
 var _ = BeforeEach(func() {

test/suites/ipv6/suite_test.go → test/suites/alpha/ipv6/suite_test.go

@@ -40,7 +40,7 @@ func TestIPv6(t *testing.T) {
 	AfterSuite(func() {
 		env.Stop()
 	})
-	RunSpecs(t, "IPv6")
+	RunSpecs(t, "Alpha/IPv6")
 }
 
 var _ = BeforeEach(func() { env.BeforeEach() })

test/suites/machine/suite_test.go → test/suites/alpha/machine/suite_test.go

@@ -33,7 +33,7 @@ func TestMachine(t *testing.T) {
 	AfterSuite(func() {
 		env.Stop()
 	})
-	RunSpecs(t, "Machine")
+	RunSpecs(t, "Alpha/Machine")
 }
 
 var _ = BeforeEach(func() { env.BeforeEach() })

test/suites/scale/suite_test.go → test/suites/alpha/scale/suite_test.go

@@ -35,7 +35,7 @@ func TestScale(t *testing.T) {
 	AfterSuite(func() {
 		env.Stop()
 	})
-	RunSpecs(t, "Scale")
+	RunSpecs(t, "Alpha/Scale")
 }
 
 var _ = BeforeEach(func() {

test/suites/utilization/suite_test.go → test/suites/alpha/utilization/suite_test.go

@@ -42,7 +42,7 @@ func TestUtilization(t *testing.T) {
 	AfterSuite(func() {
 		env.Stop()
 	})
-	RunSpecs(t, "Utilization")
+	RunSpecs(t, "Alpha/Utilization")
 }
 
 var _ = BeforeEach(func() { env.BeforeEach() })