Skip to content

test: clean up cns state check tests #2082

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jul 27, 2023
Merged

test: clean up cns state check tests #2082

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 34 additions & 8 deletions test/integration/load/load_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ package load
import (
"context"
"flag"
"fmt"
"testing"
"time"

Expand Down Expand Up @@ -35,6 +36,13 @@ var noopDeploymentMap = map[string]string{
"linux": manifestDir + "/noop-deployment-linux.yaml",
}

// Todo: Add the validation for the data path function for the linux/windows client.
type stateValidator interface {
ValidateStateFile(context.Context) error
ValidateRestartNetwork(context.Context) error
// ValidateDataPath() error
}

/*
In order to run the scale tests, you need a k8s cluster and its kubeconfig.
If no kubeconfig is passed, the test will attempt to find one in the default location for kubectl config.
Expand Down Expand Up @@ -131,21 +139,39 @@ func TestValidateState(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
defer cancel()

var validator stateValidator

t.Log("Validating the state file")
validatorClient := validate.GetValidatorClient(*osType)
validator := validatorClient.CreateClient(ctx, clientset, config, namespace, *cniType, *restartCase)
switch *osType {
case "linux":
validator, err = validate.CreateLinuxValidator(ctx, clientset, config, namespace, *cniType, *restartCase)
if err != nil {
t.Fatal(err)
}
case "windows":
validator, err = validate.CreateWindowsValidator(ctx, clientset, config, namespace, *cniType, *restartCase)
if err != nil {
t.Fatal(err)
}
default:
t.Fatalf("unknown os type %s", *osType)
}

err = validator.ValidateStateFile()
err = validator.ValidateStateFile(ctx)
if err != nil {
t.Fatal(err)
}

//We are restarting the systmemd network and checking that the connectivity works after the restart. For more details: https://github.com/cilium/cilium/issues/18706
// We are restarting the systmemd network and checking that the connectivity works after the restart. For more details: https://github.com/cilium/cilium/issues/18706
t.Log("Validating the restart network scenario")
err = validator.ValidateRestartNetwork()
if err != nil {
t.Fatal(err)
}
t.Run(fmt.Sprintf("validate network restart - %s", *osType), func(t *testing.T) {
if *osType == "windows" {
t.Skip("validate network restart not implemented on Windows")
}
if err := validator.ValidateRestartNetwork(ctx); err != nil {
t.Fatal(err)
}
})
}

// TestScaleDeployment scales the deployment up/down based on the replicas passed.
Expand Down
40 changes: 0 additions & 40 deletions test/validate/client.go
View file
Open in desktop

This file was deleted.

62 changes: 30 additions & 32 deletions test/validate/linux_validate.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,12 @@ var (

type stateFileIpsFunc func([]byte) (map[string]string, error)

type LinuxClient struct{}

type LinuxValidator struct {
Validator
clientset *kubernetes.Clientset
config *rest.Config
namespace string
cni string
restartCase bool
}

type CnsState struct {
Expand Down Expand Up @@ -62,35 +64,31 @@ type Address struct {
Addr string `json:"ipv4"`
}

func (l *LinuxClient) CreateClient(ctx context.Context, clienset *kubernetes.Clientset, config *rest.Config, namespace, cni string, restartCase bool) IValidator {
func CreateLinuxValidator(ctx context.Context, clienset *kubernetes.Clientset, config *rest.Config, namespace, cni string, restartCase bool) (*LinuxValidator, error) {
// deploy privileged pod
privilegedDaemonSet, err := k8sutils.MustParseDaemonSet(privilegedDaemonSetPath)
if err != nil {
panic(err)
return nil, errors.Wrap(err, "unable to parse daemonset")
}
daemonsetClient := clienset.AppsV1().DaemonSets(privilegedNamespace)
err = k8sutils.MustCreateDaemonset(ctx, daemonsetClient, privilegedDaemonSet)
if err != nil {
panic(err)
if err := k8sutils.MustCreateDaemonset(ctx, daemonsetClient, privilegedDaemonSet); err != nil {
return nil, errors.Wrap(err, "unable to create daemonset")
}
err = k8sutils.WaitForPodsRunning(ctx, clienset, privilegedNamespace, privilegedLabelSelector)
if err != nil {
panic(err)
if err := k8sutils.WaitForPodsRunning(ctx, clienset, privilegedNamespace, privilegedLabelSelector); err != nil {
return nil, errors.Wrap(err, "error while waiting for pods to be running")
}

return &LinuxValidator{
Validator: Validator{
ctx: ctx,
clientset: clienset,
config: config,
namespace: namespace,
cni: cni,
restartCase: restartCase,
},
}
clientset: clienset,
config: config,
namespace: namespace,
cni: cni,
restartCase: restartCase,
}, nil
}

// Todo: Based on cni version validate different state files
func (v *LinuxValidator) ValidateStateFile() error {
func (v *LinuxValidator) ValidateStateFile(ctx context.Context) error {
checkSet := make(map[string][]check) // key is cni type, value is a list of check
// TODO: add cniv1 when adding Linux related test cases
checkSet["cilium"] = []check{
Expand All @@ -104,34 +102,34 @@ func (v *LinuxValidator) ValidateStateFile() error {
}

for _, check := range checkSet[v.cni] {
err := v.validateIPs(check.stateFileIps, check.cmd, check.name, check.podNamespace, check.podLabelSelector)
err := v.validateIPs(ctx, check.stateFileIps, check.cmd, check.name, check.podNamespace, check.podLabelSelector)
if err != nil {
return err
}
}
return nil
}

func (v *LinuxValidator) ValidateRestartNetwork() error {
nodes, err := k8sutils.GetNodeList(v.ctx, v.clientset)
func (v *LinuxValidator) ValidateRestartNetwork(ctx context.Context) error {
nodes, err := k8sutils.GetNodeList(ctx, v.clientset)
if err != nil {
return errors.Wrapf(err, "failed to get node list")
}

for index := range nodes.Items {
// get the privileged pod
pod, err := k8sutils.GetPodsByNode(v.ctx, v.clientset, privilegedNamespace, privilegedLabelSelector, nodes.Items[index].Name)
pod, err := k8sutils.GetPodsByNode(ctx, v.clientset, privilegedNamespace, privilegedLabelSelector, nodes.Items[index].Name)
if err != nil {
return errors.Wrapf(err, "failed to get privileged pod")
}

privelegedPod := pod.Items[0]
// exec into the pod to get the state file
_, err = k8sutils.ExecCmdOnPod(v.ctx, v.clientset, privilegedNamespace, privelegedPod.Name, restartNetworkCmd, v.config)
_, err = k8sutils.ExecCmdOnPod(ctx, v.clientset, privilegedNamespace, privelegedPod.Name, restartNetworkCmd, v.config)
if err != nil {
return errors.Wrapf(err, "failed to exec into privileged pod")
}
err = k8sutils.WaitForPodsRunning(v.ctx, v.clientset, "", "")
err = k8sutils.WaitForPodsRunning(ctx, v.clientset, "", "")
if err != nil {
return errors.Wrapf(err, "failed to wait for pods running")
}
Expand Down Expand Up @@ -191,22 +189,22 @@ func cnsCacheStateFileIps(result []byte) (map[string]string, error) {
return cnsPodIps, nil
}

func (v *LinuxValidator) validateIPs(stateFileIps stateFileIpsFunc, cmd []string, checkType, namespace, labelSelector string) error {
func (v *LinuxValidator) validateIPs(ctx context.Context, stateFileIps stateFileIpsFunc, cmd []string, checkType, namespace, labelSelector string) error {
log.Printf("Validating %s state file", checkType)
nodes, err := k8sutils.GetNodeList(v.ctx, v.clientset)
nodes, err := k8sutils.GetNodeList(ctx, v.clientset)
if err != nil {
return errors.Wrapf(err, "failed to get node list")
}

for index := range nodes.Items {
// get the privileged pod
pod, err := k8sutils.GetPodsByNode(v.ctx, v.clientset, namespace, labelSelector, nodes.Items[index].Name)
pod, err := k8sutils.GetPodsByNode(ctx, v.clientset, namespace, labelSelector, nodes.Items[index].Name)
if err != nil {
return errors.Wrapf(err, "failed to get privileged pod")
}
podName := pod.Items[0].Name
// exec into the pod to get the state file
result, err := k8sutils.ExecCmdOnPod(v.ctx, v.clientset, namespace, podName, cmd, v.config)
result, err := k8sutils.ExecCmdOnPod(ctx, v.clientset, namespace, podName, cmd, v.config)
if err != nil {
return errors.Wrapf(err, "failed to exec into privileged pod")
}
Expand All @@ -219,7 +217,7 @@ func (v *LinuxValidator) validateIPs(stateFileIps stateFileIpsFunc, cmd []string
continue
}
// get the pod ips
podIps := getPodIPsWithoutNodeIP(v.ctx, v.clientset, nodes.Items[index])
podIps := getPodIPsWithoutNodeIP(ctx, v.clientset, nodes.Items[index])

check := compareIPs(filePodIps, podIps)

Expand Down
55 changes: 26 additions & 29 deletions test/validate/windows_validate.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,12 @@ var (
azureVnetIpamCmd = []string{"powershell", "-c", "cat ../../k/azure-vnet-ipam.json"}
)

type WindowsClient struct{}

type WindowsValidator struct {
Validator
clientset *kubernetes.Clientset
config *rest.Config
namespace string
cni string
restartCase bool
}

type HNSEndpoint struct {
Expand Down Expand Up @@ -86,34 +88,29 @@ type check struct {
cmd []string
}

func (w *WindowsClient) CreateClient(ctx context.Context, clienset *kubernetes.Clientset, config *rest.Config, namespace, cni string, restartCase bool) IValidator {
func CreateWindowsValidator(ctx context.Context, clienset *kubernetes.Clientset, config *rest.Config, namespace, cni string, restartCase bool) (*WindowsValidator, error) {
// deploy privileged pod
privilegedDaemonSet, err := k8sutils.MustParseDaemonSet(privilegedWindowsDaemonSetPath)
if err != nil {
panic(err)
return nil, errors.Wrap(err, "unable to parse daemonset")
}
daemonsetClient := clienset.AppsV1().DaemonSets(privilegedNamespace)
err = k8sutils.MustCreateDaemonset(ctx, daemonsetClient, privilegedDaemonSet)
if err != nil {
panic(err)
if err := k8sutils.MustCreateDaemonset(ctx, daemonsetClient, privilegedDaemonSet); err != nil {
return nil, errors.Wrap(err, "unable to create daemonset")
}
err = k8sutils.WaitForPodsRunning(ctx, clienset, privilegedNamespace, privilegedLabelSelector)
if err != nil {
panic(err)
if err := k8sutils.WaitForPodsRunning(ctx, clienset, privilegedNamespace, privilegedLabelSelector); err != nil {
return nil, errors.Wrap(err, "error while waiting for pods to be running")
}
return &WindowsValidator{
Validator: Validator{
ctx: ctx,
clientset: clienset,
config: config,
namespace: namespace,
cni: cni,
restartCase: restartCase,
},
}
clientset: clienset,
config: config,
namespace: namespace,
cni: cni,
restartCase: restartCase,
}, nil
}

func (v *WindowsValidator) ValidateStateFile() error {
func (v *WindowsValidator) ValidateStateFile(ctx context.Context) error {
checkSet := make(map[string][]check) // key is cni type, value is a list of check

checkSet["cniv1"] = []check{
Expand All @@ -128,7 +125,7 @@ func (v *WindowsValidator) ValidateStateFile() error {

// this is checking all IPs of the pods with the statefile
for _, check := range checkSet[v.cni] {
err := v.validateIPs(check.stateFileIps, check.cmd, check.name, check.podNamespace, check.podLabelSelector)
err := v.validateIPs(ctx, check.stateFileIps, check.cmd, check.name, check.podNamespace, check.podLabelSelector)
if err != nil {
return err
}
Expand Down Expand Up @@ -194,21 +191,21 @@ func azureVnetIpamIps(result []byte) (map[string]string, error) {
return azureVnetIpamPodIps, nil
}

func (v *WindowsValidator) validateIPs(stateFileIps stateFileIpsFunc, cmd []string, checkType, namespace, labelSelector string) error {
func (v *WindowsValidator) validateIPs(ctx context.Context, stateFileIps stateFileIpsFunc, cmd []string, checkType, namespace, labelSelector string) error {
log.Println("Validating ", checkType, " state file")
nodes, err := k8sutils.GetNodeListByLabelSelector(v.ctx, v.clientset, windowsNodeSelector)
nodes, err := k8sutils.GetNodeListByLabelSelector(ctx, v.clientset, windowsNodeSelector)
if err != nil {
return errors.Wrapf(err, "failed to get node list")
}
for index := range nodes.Items {
// get the privileged pod
pod, err := k8sutils.GetPodsByNode(v.ctx, v.clientset, namespace, labelSelector, nodes.Items[index].Name)
pod, err := k8sutils.GetPodsByNode(ctx, v.clientset, namespace, labelSelector, nodes.Items[index].Name)
if err != nil {
return errors.Wrapf(err, "failed to get privileged pod")
}
podName := pod.Items[0].Name
// exec into the pod to get the state file
result, err := k8sutils.ExecCmdOnPod(v.ctx, v.clientset, namespace, podName, cmd, v.config)
result, err := k8sutils.ExecCmdOnPod(ctx, v.clientset, namespace, podName, cmd, v.config)
if err != nil {
return errors.Wrapf(err, "failed to exec into privileged pod")
}
Expand All @@ -221,7 +218,7 @@ func (v *WindowsValidator) validateIPs(stateFileIps stateFileIpsFunc, cmd []stri
continue
}
// get the pod ips
podIps := getPodIPsWithoutNodeIP(v.ctx, v.clientset, nodes.Items[index])
podIps := getPodIPsWithoutNodeIP(ctx, v.clientset, nodes.Items[index])

check := compareIPs(filePodIps, podIps)

Expand All @@ -233,6 +230,6 @@ func (v *WindowsValidator) validateIPs(stateFileIps stateFileIpsFunc, cmd []stri
return nil
}

func (v *WindowsValidator) ValidateRestartNetwork() error {
return nil
func (v *WindowsValidator) ValidateRestartNetwork(context.Context) error {
return errors.New("not implemented")
}