From 27c37c5ef1df1447effd82e5b47b62a648d10a90 Mon Sep 17 00:00:00 2001 From: jpayne3506 Date: Sat, 30 Dec 2023 00:52:51 -0800 Subject: [PATCH 1/2] ci: improve TestValidateState --- test/integration/load/load_test.go | 57 ++++++++++++++++++++++++--- test/internal/kubernetes/utils.go | 12 ++++++ test/internal/kubernetes/utils_get.go | 10 +++++ 3 files changed, 74 insertions(+), 5 deletions(-) diff --git a/test/integration/load/load_test.go b/test/integration/load/load_test.go index 2a6388d1a4..6822428986 100644 --- a/test/integration/load/load_test.go +++ b/test/integration/load/load_test.go @@ -124,6 +124,46 @@ func TestValidateState(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) defer cancel() + if testConfig.ValidateStateFile { + deployment := kubernetes.MustParseDeployment(noopDeploymentMap[testConfig.OSType]) + deploymentsClient := clientset.AppsV1().Deployments(namespace) + + // Ensure pods exist on nodes to validate state files properly. Can obtain false positives without pods. + nodes, err := kubernetes.GetNodeListByLabelSelector(ctx, clientset, "kubernetes.io/os="+testConfig.OSType) + require.NoError(t, err) + nodeCount := len(nodes.Items) + replicas := int32(nodeCount) * 2 + + deploymentExists, err := kubernetes.DeploymentExists(ctx, deploymentsClient, deployment.Name) + require.NoError(t, err) + if !deploymentExists { + t.Logf("Test deployment %s does not exist! Create %v pods in %s namespace", deployment.Name, replicas, namespace) + // Create namespace if it doesn't exist + namespaceExists, err := kubernetes.NamespaceExists(ctx, clientset, namespace) + require.NoError(t, err) + if !namespaceExists { + kubernetes.MustCreateNamespace(ctx, clientset, namespace) + } + + kubernetes.MustCreateDeployment(ctx, deploymentsClient, deployment) + kubernetes.MustScaleDeployment(ctx, deploymentsClient, deployment, clientset, namespace, podLabelSelector, int(replicas), false) + } else { + t.Log("Test deployment exists! Use existing setup") + replicas, err = kubernetes.GetDeploymentAvailableReplicas(ctx, deploymentsClient, deployment.Name) // If test namespace exists then use existing Replicas + if replicas != 0 && err != nil { + require.NoError(t, err) + } + } + if replicas < int32(nodeCount) { + t.Logf("Warning - current replica count %v is below current %s node count of %d. Raising replicas to minimum required to ensure there is a pod on every node.", replicas, testConfig.OSType, nodeCount) + replicas = int32(nodeCount * 2) + kubernetes.MustScaleDeployment(ctx, deploymentsClient, deployment, clientset, namespace, podLabelSelector, int(replicas), false) + } + t.Log("Ensure deployment is in ready status") + err = kubernetes.WaitForPodDeployment(ctx, clientset, namespace, deployment.Name, podLabelSelector, int(replicas)) + require.NoError(t, err) + } + validator, err := validate.CreateValidator(ctx, clientset, config, namespace, testConfig.CNIType, testConfig.RestartCase, testConfig.OSType) require.NoError(t, err) @@ -176,10 +216,12 @@ func TestValidCNSStateDuringScaleAndCNSRestartToTriggerDropgzInstall(t *testing. ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) defer cancel() - validator, err := validate.CreateValidator(ctx, clientset, config, namespace, testConfig.CNIType, testConfig.RestartCase, testConfig.OSType) - require.NoError(t, err) + // Provide option to validate state before running test + if testConfig.ValidateStateFile { + t.Run("Validate state file", TestValidateState) + } - err = validator.Validate(ctx) + validator, err := validate.CreateValidator(ctx, clientset, config, namespace, testConfig.CNIType, testConfig.RestartCase, testConfig.OSType) require.NoError(t, err) deployment := kubernetes.MustParseDeployment(noopDeploymentMap[testConfig.OSType]) @@ -218,8 +260,9 @@ func TestValidCNSStateDuringScaleAndCNSRestartToTriggerDropgzInstall(t *testing. require.NoError(t, err) // Validate the CNS state - err = validator.Validate(ctx) - require.NoError(t, err) + if testConfig.ValidateStateFile { + t.Run("Validate state file", TestValidateState) + } if testConfig.Cleanup { kubernetes.MustDeleteDeployment(ctx, deploymentsClient, deployment) @@ -245,6 +288,10 @@ func TestV4OverlayProperties(t *testing.T) { t.Log("Validating v4Overlay node labels") err = validator.ValidateV4OverlayControlPlane(ctx) require.NoError(t, err) + + if testConfig.Cleanup { + validator.Cleanup(ctx) + } } func TestDualStackProperties(t *testing.T) { diff --git a/test/internal/kubernetes/utils.go b/test/internal/kubernetes/utils.go index 16883f66ff..d8c3abe85e 100644 --- a/test/internal/kubernetes/utils.go +++ b/test/internal/kubernetes/utils.go @@ -413,6 +413,18 @@ func NamespaceExists(ctx context.Context, clientset *kubernetes.Clientset, names return true, nil } +func DeploymentExists(ctx context.Context, deploymentsClient typedappsv1.DeploymentInterface, deploymentName string) (bool, error) { + _, err := deploymentsClient.Get(ctx, deploymentName, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, errors.Wrapf(err, "error in getting deployment %s", deploymentName) + } + + return true, nil +} + // return a label selector func CreateLabelSelector(key string, selector *string) string { return fmt.Sprintf("%s=%s", key, *selector) diff --git a/test/internal/kubernetes/utils_get.go b/test/internal/kubernetes/utils_get.go index ce4e569362..a6fecd597e 100644 --- a/test/internal/kubernetes/utils_get.go +++ b/test/internal/kubernetes/utils_get.go @@ -7,6 +7,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" + typedappsv1 "k8s.io/client-go/kubernetes/typed/apps/v1" ) func GetNodeList(ctx context.Context, clientset *kubernetes.Clientset) (*corev1.NodeList, error) { @@ -51,3 +52,12 @@ func GetPodsIpsByNode(ctx context.Context, clientset *kubernetes.Clientset, name } return ips, nil } + +func GetDeploymentAvailableReplicas(ctx context.Context, deploymentsClient typedappsv1.DeploymentInterface, deploymentName string) (int32, error) { + deployment, err := deploymentsClient.Get(ctx, deploymentName, metav1.GetOptions{}) + if err != nil { + return -1, errors.Wrapf(err, "could not get deployment %s", deploymentName) + } + + return deployment.Status.AvailableReplicas, nil +} From d29b2a737cd2bc37c4f5a4f027ea56dcb86389d0 Mon Sep 17 00:00:00 2001 From: jpayne3506 Date: Wed, 10 Jan 2024 22:34:58 -0800 Subject: [PATCH 2/2] chore: address comments --- test/integration/load/load_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/integration/load/load_test.go b/test/integration/load/load_test.go index 6822428986..a9449672de 100644 --- a/test/integration/load/load_test.go +++ b/test/integration/load/load_test.go @@ -216,7 +216,7 @@ func TestValidCNSStateDuringScaleAndCNSRestartToTriggerDropgzInstall(t *testing. ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) defer cancel() - // Provide option to validate state before running test + // Provide an option to validate state files with a proper environment before running test if testConfig.ValidateStateFile { t.Run("Validate state file", TestValidateState) } @@ -260,14 +260,14 @@ func TestValidCNSStateDuringScaleAndCNSRestartToTriggerDropgzInstall(t *testing. require.NoError(t, err) // Validate the CNS state - if testConfig.ValidateStateFile { - t.Run("Validate state file", TestValidateState) - } + err = validator.Validate(ctx) + require.NoError(t, err) if testConfig.Cleanup { kubernetes.MustDeleteDeployment(ctx, deploymentsClient, deployment) err = kubernetes.WaitForPodsDelete(ctx, clientset, namespace, podLabelSelector) require.NoError(t, err, "error waiting for pods to delete") + validator.Cleanup(ctx) } }