From 19682cf4f05e8f684180be4dcfbd1e1dfbb60c78 Mon Sep 17 00:00:00 2001 From: Matthew Long <61910737+thatmattlong@users.noreply.github.com> Date: Wed, 26 Jul 2023 15:40:20 -0700 Subject: [PATCH] test: clean up cns state check tests --- test/integration/load/load_test.go | 42 ++++++++++++++++---- test/validate/client.go | 40 ------------------- test/validate/linux_validate.go | 62 +++++++++++++++--------------- test/validate/windows_validate.go | 55 +++++++++++++------------- 4 files changed, 90 insertions(+), 109 deletions(-) delete mode 100644 test/validate/client.go diff --git a/test/integration/load/load_test.go b/test/integration/load/load_test.go index ae09235eac..e84c0bcc80 100644 --- a/test/integration/load/load_test.go +++ b/test/integration/load/load_test.go @@ -5,6 +5,7 @@ package load import ( "context" "flag" + "fmt" "testing" "time" @@ -35,6 +36,13 @@ var noopDeploymentMap = map[string]string{ "linux": manifestDir + "/noop-deployment-linux.yaml", } +// Todo: Add the validation for the data path function for the linux/windows client. +type stateValidator interface { + ValidateStateFile(context.Context) error + ValidateRestartNetwork(context.Context) error + // ValidateDataPath() error +} + /* In order to run the scale tests, you need a k8s cluster and its kubeconfig. If no kubeconfig is passed, the test will attempt to find one in the default location for kubectl config. @@ -131,21 +139,39 @@ func TestValidateState(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) defer cancel() + var validator stateValidator + t.Log("Validating the state file") - validatorClient := validate.GetValidatorClient(*osType) - validator := validatorClient.CreateClient(ctx, clientset, config, namespace, *cniType, *restartCase) + switch *osType { + case "linux": + validator, err = validate.CreateLinuxValidator(ctx, clientset, config, namespace, *cniType, *restartCase) + if err != nil { + t.Fatal(err) + } + case "windows": + validator, err = validate.CreateWindowsValidator(ctx, clientset, config, namespace, *cniType, *restartCase) + if err != nil { + t.Fatal(err) + } + default: + t.Fatalf("unknown os type %s", *osType) + } - err = validator.ValidateStateFile() + err = validator.ValidateStateFile(ctx) if err != nil { t.Fatal(err) } - //We are restarting the systmemd network and checking that the connectivity works after the restart. For more details: https://github.com/cilium/cilium/issues/18706 + // We are restarting the systmemd network and checking that the connectivity works after the restart. For more details: https://github.com/cilium/cilium/issues/18706 t.Log("Validating the restart network scenario") - err = validator.ValidateRestartNetwork() - if err != nil { - t.Fatal(err) - } + t.Run(fmt.Sprintf("validate network restart - %s", *osType), func(t *testing.T) { + if *osType == "windows" { + t.Skip("validate network restart not implemented on Windows") + } + if err := validator.ValidateRestartNetwork(ctx); err != nil { + t.Fatal(err) + } + }) } // TestScaleDeployment scales the deployment up/down based on the replicas passed. diff --git a/test/validate/client.go b/test/validate/client.go deleted file mode 100644 index b4cc5f5cb8..0000000000 --- a/test/validate/client.go +++ /dev/null @@ -1,40 +0,0 @@ -package validate - -import ( - "context" - - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" -) - -type Validator struct { - ctx context.Context - clientset *kubernetes.Clientset - config *rest.Config - namespace string - cni string - restartCase bool -} - -// Todo: Add the validation for the data path function for the linux/windows client. -type IValidator interface { - ValidateStateFile() error - ValidateRestartNetwork() error - // ValidateDataPath() error -} - -type validatorClient interface { - CreateClient(ctx context.Context, clienset *kubernetes.Clientset, config *rest.Config, namespace, cni string, restartCase bool) IValidator -} - -// Func to get the type of validator client based on the Operating system. -func GetValidatorClient(os string) validatorClient { - switch os { - case "linux": - return &LinuxClient{} - case "windows": - return &WindowsClient{} - default: - return nil - } -} diff --git a/test/validate/linux_validate.go b/test/validate/linux_validate.go index 99fe2a0e3e..d97a8b3f5b 100644 --- a/test/validate/linux_validate.go +++ b/test/validate/linux_validate.go @@ -31,10 +31,12 @@ var ( type stateFileIpsFunc func([]byte) (map[string]string, error) -type LinuxClient struct{} - type LinuxValidator struct { - Validator + clientset *kubernetes.Clientset + config *rest.Config + namespace string + cni string + restartCase bool } type CnsState struct { @@ -62,35 +64,31 @@ type Address struct { Addr string `json:"ipv4"` } -func (l *LinuxClient) CreateClient(ctx context.Context, clienset *kubernetes.Clientset, config *rest.Config, namespace, cni string, restartCase bool) IValidator { +func CreateLinuxValidator(ctx context.Context, clienset *kubernetes.Clientset, config *rest.Config, namespace, cni string, restartCase bool) (*LinuxValidator, error) { // deploy privileged pod privilegedDaemonSet, err := k8sutils.MustParseDaemonSet(privilegedDaemonSetPath) if err != nil { - panic(err) + return nil, errors.Wrap(err, "unable to parse daemonset") } daemonsetClient := clienset.AppsV1().DaemonSets(privilegedNamespace) - err = k8sutils.MustCreateDaemonset(ctx, daemonsetClient, privilegedDaemonSet) - if err != nil { - panic(err) + if err := k8sutils.MustCreateDaemonset(ctx, daemonsetClient, privilegedDaemonSet); err != nil { + return nil, errors.Wrap(err, "unable to create daemonset") } - err = k8sutils.WaitForPodsRunning(ctx, clienset, privilegedNamespace, privilegedLabelSelector) - if err != nil { - panic(err) + if err := k8sutils.WaitForPodsRunning(ctx, clienset, privilegedNamespace, privilegedLabelSelector); err != nil { + return nil, errors.Wrap(err, "error while waiting for pods to be running") } + return &LinuxValidator{ - Validator: Validator{ - ctx: ctx, - clientset: clienset, - config: config, - namespace: namespace, - cni: cni, - restartCase: restartCase, - }, - } + clientset: clienset, + config: config, + namespace: namespace, + cni: cni, + restartCase: restartCase, + }, nil } // Todo: Based on cni version validate different state files -func (v *LinuxValidator) ValidateStateFile() error { +func (v *LinuxValidator) ValidateStateFile(ctx context.Context) error { checkSet := make(map[string][]check) // key is cni type, value is a list of check // TODO: add cniv1 when adding Linux related test cases checkSet["cilium"] = []check{ @@ -104,7 +102,7 @@ func (v *LinuxValidator) ValidateStateFile() error { } for _, check := range checkSet[v.cni] { - err := v.validateIPs(check.stateFileIps, check.cmd, check.name, check.podNamespace, check.podLabelSelector) + err := v.validateIPs(ctx, check.stateFileIps, check.cmd, check.name, check.podNamespace, check.podLabelSelector) if err != nil { return err } @@ -112,26 +110,26 @@ func (v *LinuxValidator) ValidateStateFile() error { return nil } -func (v *LinuxValidator) ValidateRestartNetwork() error { - nodes, err := k8sutils.GetNodeList(v.ctx, v.clientset) +func (v *LinuxValidator) ValidateRestartNetwork(ctx context.Context) error { + nodes, err := k8sutils.GetNodeList(ctx, v.clientset) if err != nil { return errors.Wrapf(err, "failed to get node list") } for index := range nodes.Items { // get the privileged pod - pod, err := k8sutils.GetPodsByNode(v.ctx, v.clientset, privilegedNamespace, privilegedLabelSelector, nodes.Items[index].Name) + pod, err := k8sutils.GetPodsByNode(ctx, v.clientset, privilegedNamespace, privilegedLabelSelector, nodes.Items[index].Name) if err != nil { return errors.Wrapf(err, "failed to get privileged pod") } privelegedPod := pod.Items[0] // exec into the pod to get the state file - _, err = k8sutils.ExecCmdOnPod(v.ctx, v.clientset, privilegedNamespace, privelegedPod.Name, restartNetworkCmd, v.config) + _, err = k8sutils.ExecCmdOnPod(ctx, v.clientset, privilegedNamespace, privelegedPod.Name, restartNetworkCmd, v.config) if err != nil { return errors.Wrapf(err, "failed to exec into privileged pod") } - err = k8sutils.WaitForPodsRunning(v.ctx, v.clientset, "", "") + err = k8sutils.WaitForPodsRunning(ctx, v.clientset, "", "") if err != nil { return errors.Wrapf(err, "failed to wait for pods running") } @@ -191,22 +189,22 @@ func cnsCacheStateFileIps(result []byte) (map[string]string, error) { return cnsPodIps, nil } -func (v *LinuxValidator) validateIPs(stateFileIps stateFileIpsFunc, cmd []string, checkType, namespace, labelSelector string) error { +func (v *LinuxValidator) validateIPs(ctx context.Context, stateFileIps stateFileIpsFunc, cmd []string, checkType, namespace, labelSelector string) error { log.Printf("Validating %s state file", checkType) - nodes, err := k8sutils.GetNodeList(v.ctx, v.clientset) + nodes, err := k8sutils.GetNodeList(ctx, v.clientset) if err != nil { return errors.Wrapf(err, "failed to get node list") } for index := range nodes.Items { // get the privileged pod - pod, err := k8sutils.GetPodsByNode(v.ctx, v.clientset, namespace, labelSelector, nodes.Items[index].Name) + pod, err := k8sutils.GetPodsByNode(ctx, v.clientset, namespace, labelSelector, nodes.Items[index].Name) if err != nil { return errors.Wrapf(err, "failed to get privileged pod") } podName := pod.Items[0].Name // exec into the pod to get the state file - result, err := k8sutils.ExecCmdOnPod(v.ctx, v.clientset, namespace, podName, cmd, v.config) + result, err := k8sutils.ExecCmdOnPod(ctx, v.clientset, namespace, podName, cmd, v.config) if err != nil { return errors.Wrapf(err, "failed to exec into privileged pod") } @@ -219,7 +217,7 @@ func (v *LinuxValidator) validateIPs(stateFileIps stateFileIpsFunc, cmd []string continue } // get the pod ips - podIps := getPodIPsWithoutNodeIP(v.ctx, v.clientset, nodes.Items[index]) + podIps := getPodIPsWithoutNodeIP(ctx, v.clientset, nodes.Items[index]) check := compareIPs(filePodIps, podIps) diff --git a/test/validate/windows_validate.go b/test/validate/windows_validate.go index ab9a0f2aed..6ce997a173 100644 --- a/test/validate/windows_validate.go +++ b/test/validate/windows_validate.go @@ -23,10 +23,12 @@ var ( azureVnetIpamCmd = []string{"powershell", "-c", "cat ../../k/azure-vnet-ipam.json"} ) -type WindowsClient struct{} - type WindowsValidator struct { - Validator + clientset *kubernetes.Clientset + config *rest.Config + namespace string + cni string + restartCase bool } type HNSEndpoint struct { @@ -86,34 +88,29 @@ type check struct { cmd []string } -func (w *WindowsClient) CreateClient(ctx context.Context, clienset *kubernetes.Clientset, config *rest.Config, namespace, cni string, restartCase bool) IValidator { +func CreateWindowsValidator(ctx context.Context, clienset *kubernetes.Clientset, config *rest.Config, namespace, cni string, restartCase bool) (*WindowsValidator, error) { // deploy privileged pod privilegedDaemonSet, err := k8sutils.MustParseDaemonSet(privilegedWindowsDaemonSetPath) if err != nil { - panic(err) + return nil, errors.Wrap(err, "unable to parse daemonset") } daemonsetClient := clienset.AppsV1().DaemonSets(privilegedNamespace) - err = k8sutils.MustCreateDaemonset(ctx, daemonsetClient, privilegedDaemonSet) - if err != nil { - panic(err) + if err := k8sutils.MustCreateDaemonset(ctx, daemonsetClient, privilegedDaemonSet); err != nil { + return nil, errors.Wrap(err, "unable to create daemonset") } - err = k8sutils.WaitForPodsRunning(ctx, clienset, privilegedNamespace, privilegedLabelSelector) - if err != nil { - panic(err) + if err := k8sutils.WaitForPodsRunning(ctx, clienset, privilegedNamespace, privilegedLabelSelector); err != nil { + return nil, errors.Wrap(err, "error while waiting for pods to be running") } return &WindowsValidator{ - Validator: Validator{ - ctx: ctx, - clientset: clienset, - config: config, - namespace: namespace, - cni: cni, - restartCase: restartCase, - }, - } + clientset: clienset, + config: config, + namespace: namespace, + cni: cni, + restartCase: restartCase, + }, nil } -func (v *WindowsValidator) ValidateStateFile() error { +func (v *WindowsValidator) ValidateStateFile(ctx context.Context) error { checkSet := make(map[string][]check) // key is cni type, value is a list of check checkSet["cniv1"] = []check{ @@ -128,7 +125,7 @@ func (v *WindowsValidator) ValidateStateFile() error { // this is checking all IPs of the pods with the statefile for _, check := range checkSet[v.cni] { - err := v.validateIPs(check.stateFileIps, check.cmd, check.name, check.podNamespace, check.podLabelSelector) + err := v.validateIPs(ctx, check.stateFileIps, check.cmd, check.name, check.podNamespace, check.podLabelSelector) if err != nil { return err } @@ -194,21 +191,21 @@ func azureVnetIpamIps(result []byte) (map[string]string, error) { return azureVnetIpamPodIps, nil } -func (v *WindowsValidator) validateIPs(stateFileIps stateFileIpsFunc, cmd []string, checkType, namespace, labelSelector string) error { +func (v *WindowsValidator) validateIPs(ctx context.Context, stateFileIps stateFileIpsFunc, cmd []string, checkType, namespace, labelSelector string) error { log.Println("Validating ", checkType, " state file") - nodes, err := k8sutils.GetNodeListByLabelSelector(v.ctx, v.clientset, windowsNodeSelector) + nodes, err := k8sutils.GetNodeListByLabelSelector(ctx, v.clientset, windowsNodeSelector) if err != nil { return errors.Wrapf(err, "failed to get node list") } for index := range nodes.Items { // get the privileged pod - pod, err := k8sutils.GetPodsByNode(v.ctx, v.clientset, namespace, labelSelector, nodes.Items[index].Name) + pod, err := k8sutils.GetPodsByNode(ctx, v.clientset, namespace, labelSelector, nodes.Items[index].Name) if err != nil { return errors.Wrapf(err, "failed to get privileged pod") } podName := pod.Items[0].Name // exec into the pod to get the state file - result, err := k8sutils.ExecCmdOnPod(v.ctx, v.clientset, namespace, podName, cmd, v.config) + result, err := k8sutils.ExecCmdOnPod(ctx, v.clientset, namespace, podName, cmd, v.config) if err != nil { return errors.Wrapf(err, "failed to exec into privileged pod") } @@ -221,7 +218,7 @@ func (v *WindowsValidator) validateIPs(stateFileIps stateFileIpsFunc, cmd []stri continue } // get the pod ips - podIps := getPodIPsWithoutNodeIP(v.ctx, v.clientset, nodes.Items[index]) + podIps := getPodIPsWithoutNodeIP(ctx, v.clientset, nodes.Items[index]) check := compareIPs(filePodIps, podIps) @@ -233,6 +230,6 @@ func (v *WindowsValidator) validateIPs(stateFileIps stateFileIpsFunc, cmd []stri return nil } -func (v *WindowsValidator) ValidateRestartNetwork() error { - return nil +func (v *WindowsValidator) ValidateRestartNetwork(context.Context) error { + return errors.New("not implemented") }