From 7ac3b4d8eb11e4a25879d75e4d1153edff0d9cba Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Wed, 21 Jun 2023 21:45:40 -0400 Subject: [PATCH 01/45] add Linux test cases for connectivity --- .../datapath/datapath_linux_test.go | 238 ++++++++++++++++++ .../manifests/datapath/linux-deployment.yaml | 74 ++++++ test/internal/datapath/datapath_linux.go | 167 ++++++++++++ test/internal/k8sutils/utils.go | 1 - 4 files changed, 479 insertions(+), 1 deletion(-) create mode 100644 test/integration/datapath/datapath_linux_test.go create mode 100644 test/integration/manifests/datapath/linux-deployment.yaml create mode 100644 test/internal/datapath/datapath_linux.go diff --git a/test/integration/datapath/datapath_linux_test.go b/test/integration/datapath/datapath_linux_test.go new file mode 100644 index 0000000000..acf9cea2f1 --- /dev/null +++ b/test/integration/datapath/datapath_linux_test.go @@ -0,0 +1,238 @@ +//go:build connection + +package connection + +import ( + "context" + "flag" + "fmt" + "testing" + "time" + + "github.com/Azure/azure-container-networking/test/integration" + "github.com/Azure/azure-container-networking/test/integration/goldpinger" + k8sutils "github.com/Azure/azure-container-networking/test/internal/k8sutils" + "github.com/Azure/azure-container-networking/test/internal/retry" + "github.com/pkg/errors" + "github.com/stretchr/testify/require" + + apiv1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + LinuxDeployYamlPath = "../manifests/datapath/linux-deployment.yaml" + podLabelKey = "app" + podCount = 2 + nodepoolKey = "agentpool" + maxRetryDelaySeconds = 10 + defaultTimeoutSeconds = 120 + defaultRetryDelaySeconds = 1 + goldpingerRetryCount = 24 + goldpingerDelayTimeSeconds = 5 +) + +var ( + podPrefix = flag.String("podName", "goldpinger", "Prefix for test pods") + podNamespace = flag.String("namespace", "datapath-linux", "Namespace for test pods") + nodepoolSelector = flag.String("nodepoolSelector", "nodepool1", "Provides nodepool as a Node-Selector for pods") + defaultRetrier = retry.Retrier{ + Attempts: 10, + Delay: defaultRetryDelaySeconds * time.Second, + } +) + +/* +This test assumes that you have the current credentials loaded in your default kubeconfig for a +k8s cluster with a Linux nodepool consisting of at least 2 Linux nodes. +*** The expected nodepool name is npwin, if the nodepool has a diferent name ensure that you change nodepoolSelector with: + -nodepoolSelector="yournodepoolname" + +To run the test use one of the following commands: +go test -count=1 test/integration/datapath/datapath_linux_test.go -timeout 3m -tags connection -run ^TestDatapathLinux$ -tags=connection + or +go test -count=1 test/integration/datapath/datapath_linux_test.go -timeout 3m -tags connection -run ^TestDatapathLinux$ -podName=acnpod -nodepoolSelector=npwina -tags=connection + + +This test checks pod to pod, pod to node, and pod to internet for datapath connectivity. + +Timeout context is controled by the -timeout flag. + +*/ + +func TestDatapathLinux(t *testing.T) { + ctx := context.Background() + + t.Log("Create Clientset") + clientset, err := k8sutils.MustGetClientset() + if err != nil { + require.NoError(t, err, "could not get k8s clientset: %v", err) + } + t.Log("Get REST config") + restConfig := k8sutils.MustGetRestConfig(t) + + t.Log("Create Label Selectors") + + podLabelSelector := fmt.Sprintf("%s=%s", podLabelKey, *podPrefix) + nodeLabelSelector := fmt.Sprintf("%s=%s", nodepoolKey, *nodepoolSelector) + + t.Log("Get Nodes") + nodes, err := k8sutils.GetNodeListByLabelSelector(ctx, clientset, nodeLabelSelector) + if err != nil { + require.NoError(t, err, "could not get k8s node list: %v", err) + } + + // Test Namespace + t.Log("Create Namespace") + err = k8sutils.MustCreateNamespace(ctx, clientset, *podNamespace) + createPodFlag := !(apierrors.IsAlreadyExists(err)) + t.Logf("%v", createPodFlag) + + if createPodFlag { + t.Log("Creating Linux pods through deployment") + deployment, err := k8sutils.MustParseDeployment(LinuxDeployYamlPath) + if err != nil { + require.NoError(t, err) + } + + // Fields for overwritting existing deployment yaml. + // Defaults from flags will not change anything + deployment.Spec.Selector.MatchLabels[podLabelKey] = *podPrefix + deployment.Spec.Template.ObjectMeta.Labels[podLabelKey] = *podPrefix + deployment.Spec.Template.Spec.NodeSelector[nodepoolKey] = *nodepoolSelector + deployment.Name = *podPrefix + deployment.Namespace = *podNamespace + + t.Logf("deployment Spec Template is %+v", deployment.Spec.Template) + deploymentsClient := clientset.AppsV1().Deployments(*podNamespace) + err = k8sutils.MustCreateDeployment(ctx, deploymentsClient, deployment) + if err != nil { + require.NoError(t, err) + } + t.Logf("podNamespace is %s", *podNamespace) + t.Logf("podLabelSelector is %s", podLabelSelector) + + t.Log("Waiting for pods to be running state") + err = k8sutils.WaitForPodsRunning(ctx, clientset, *podNamespace, podLabelSelector) + if err != nil { + require.NoError(t, err) + } + t.Log("Successfully created customer linux pods") + } else { + // Checks namespace already exists from previous attempt + t.Log("Namespace already exists") + + t.Log("Checking for pods to be running state") + err = k8sutils.WaitForPodsRunning(ctx, clientset, *podNamespace, podLabelSelector) + if err != nil { + require.NoError(t, err) + } + } + t.Log("Checking Linux test environment") + for _, node := range nodes.Items { + + pods, err := k8sutils.GetPodsByNode(ctx, clientset, *podNamespace, podLabelSelector, node.Name) + if err != nil { + require.NoError(t, err, "could not get k8s clientset: %v", err) + } + if len(pods.Items) <= 1 { + t.Logf("%s", node.Name) + require.NoError(t, errors.New("Less than 2 pods on node")) + } + } + t.Log("Linux test environment ready") + + t.Run("Linux ping tests", func(t *testing.T) { + // Check goldpinger health + t.Run("all pods have IPs assigned", func(t *testing.T) { + podsClient := clientset.CoreV1().Pods(*podNamespace) + + checkPodIPsFn := func() error { + podList, err := podsClient.List(ctx, metav1.ListOptions{LabelSelector: "app=goldpinger"}) + t.Logf("podList is %+v", podList) + if err != nil { + return err + } + + if len(podList.Items) == 0 { + return errors.New("no pods scheduled") + } + + for _, pod := range podList.Items { + if pod.Status.Phase == apiv1.PodPending { + return errors.New("some pods still pending") + } + } + + for _, pod := range podList.Items { + if pod.Status.PodIP == "" { + return errors.New("a pod has not been allocated an IP") + } + } + + return nil + } + err := defaultRetrier.Do(ctx, checkPodIPsFn) + if err != nil { + t.Fatalf("not all pods were allocated IPs: %v", err) + } + t.Log("all pods have been allocated IPs") + }) + + t.Run("all linux pods can ping each other", func(t *testing.T) { + pfOpts := k8s.PortForwardingOpts{ + Namespace: "default", + LabelSelector: "type=goldpinger-pod", + LocalPort: 9090, + DestPort: 8080, + } + + pf, err := k8s.NewPortForwarder(restConfig, t, pfOpts) + if err != nil { + t.Fatal(err) + } + + portForwardCtx, cancel := context.WithTimeout(ctx, defaultTimeoutSeconds*time.Second) + defer cancel() + + portForwardFn := func() error { + err := pf.Forward(portForwardCtx) + if err != nil { + t.Logf("unable to start port forward: %v", err) + return err + } + return nil + } + if err := defaultRetrier.Do(portForwardCtx, portForwardFn); err != nil { + t.Fatalf("could not start port forward within %ds: %v", defaultTimeoutSeconds, err) + } + defer pf.Stop() + + gpClient := goldpinger.Client{Host: pf.Address()} + + clusterCheckCtx, cancel := context.WithTimeout(ctx, 3*time.Minute) + defer cancel() + clusterCheckFn := func() error { + clusterState, err := gpClient.CheckAll(clusterCheckCtx) + if err != nil { + return err + } + + stats := goldpinger.ClusterStats(clusterState) + stats.PrintStats() + if stats.AllPingsHealthy() { + return nil + } + + return errors.New("not all pings are healthy") + } + retrier := retry.Retrier{Attempts: goldpingerRetryCount, Delay: goldpingerDelayTimeSeconds * time.Second} + if err := retrier.Do(clusterCheckCtx, clusterCheckFn); err != nil { + t.Fatalf("goldpinger pods network health could not reach healthy state after %d seconds: %v", goldpingerRetryCount*goldpingerDelayTimeSeconds, err) + } + + t.Log("all pings successful!") + }) + }) +} diff --git a/test/integration/manifests/datapath/linux-deployment.yaml b/test/integration/manifests/datapath/linux-deployment.yaml new file mode 100644 index 0000000000..4b029c28c3 --- /dev/null +++ b/test/integration/manifests/datapath/linux-deployment.yaml @@ -0,0 +1,74 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: goldpinger-deploy + namespace: datapath-linux +spec: + replicas: 4 + selector: + matchLabels: + app: goldpinger + template: + metadata: + labels: + app: goldpinger + spec: + serviceAccount: "goldpinger-serviceaccount" + securityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 2000 + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: "app" + operator: In + values: + - goldpinger + topologyKey: "kubernetes.io/hostname" + containers: + - name: goldpinger + env: + - name: HOST + value: "0.0.0.0" + - name: PORT + value: "8080" + - name: HOSTNAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: "docker.io/bloomberg/goldpinger:v3.0.0" + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + resources: + limits: + memory: 80Mi + requests: + cpu: 1m + memory: 40Mi + ports: + - containerPort: 8080 + name: http + readinessProbe: + httpGet: + path: /healthz + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 5 + livenessProbe: + httpGet: + path: /healthz + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 5 + nodeSelector: + kubernetes.io/os: linux diff --git a/test/internal/datapath/datapath_linux.go b/test/internal/datapath/datapath_linux.go new file mode 100644 index 0000000000..f32d4cda3f --- /dev/null +++ b/test/internal/datapath/datapath_linux.go @@ -0,0 +1,167 @@ +package datapath + +import ( + "context" + "fmt" + + "github.com/Azure/azure-container-networking/test/internal/k8sutils" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" +) + +func LinuxPodToPodPingTestSameNode(ctx context.Context, clientset *kubernetes.Clientset, nodeName, podNamespace, labelSelector string, rc *restclient.Config) error { + logrus.Infof("Get Pods for Linux Node: %s", nodeName) + pods, err := k8sutils.GetPodsByNode(ctx, clientset, podNamespace, labelSelector, nodeName) + if err != nil { + logrus.Error(err) + return errors.Wrap(err, "k8s api call") + } + if len(pods.Items) <= 1 { + return errors.New("Less than 2 pods on node") + } + + // Get first pod on this node + firstPod, err := clientset.CoreV1().Pods(podNamespace).Get(ctx, pods.Items[0].Name, metav1.GetOptions{}) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("Getting pod %s failed with %v", firstPod.Name, err)) + } + logrus.Infof("First pod: %v %v", firstPod.Name, firstPod.Status.PodIP) + + // Get the second pod on this node + secondPod, err := clientset.CoreV1().Pods(podNamespace).Get(ctx, pods.Items[1].Name, metav1.GetOptions{}) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("Getting pod %s failed with %v", secondPod.Name, err)) + } + logrus.Infof("Second pod: %v %v", secondPod.Name, secondPod.Status.PodIP) + + // Ping the second pod from the first pod + return podTest(ctx, clientset, firstPod, []string{"ping", secondPod.Status.PodIP}, rc, pingPassedWindows) +} + +// func WindowsPodToPodPingTestDiffNode(ctx context.Context, clientset *kubernetes.Clientset, nodeName1, nodeName2, podNamespace, labelSelector string, rc *restclient.Config) error { +// logrus.Infof("Get Pods for Node 1: %s", nodeName1) +// // Node 1 +// pods, err := k8sutils.GetPodsByNode(ctx, clientset, podNamespace, labelSelector, nodeName1) +// if err != nil { +// logrus.Error(err) +// return errors.Wrap(err, "k8s api call") +// } +// firstPod, err := clientset.CoreV1().Pods(podNamespace).Get(ctx, pods.Items[0].Name, metav1.GetOptions{}) +// if err != nil { +// return errors.Wrap(err, fmt.Sprintf("Getting pod %s failed with %v", firstPod.Name, err)) +// } +// logrus.Infof("First pod: %v %v", firstPod.Name, firstPod.Status.PodIP) + +// logrus.Infof("Get Pods for Node 2: %s", nodeName2) +// // Node 2 +// pods, err = k8sutils.GetPodsByNode(ctx, clientset, podNamespace, labelSelector, nodeName2) +// if err != nil { +// logrus.Error(err) +// return errors.Wrap(err, "k8s api call") +// } +// secondPod, err := clientset.CoreV1().Pods(podNamespace).Get(ctx, pods.Items[0].Name, metav1.GetOptions{}) +// if err != nil { +// return errors.Wrap(err, fmt.Sprintf("Getting pod %s failed with %v", secondPod.Name, err)) +// } +// logrus.Infof("Second pod: %v %v", secondPod.Name, secondPod.Status.PodIP) + +// // Ping the second pod from the first pod located on different nodes +// return podTest(ctx, clientset, firstPod, []string{"ping", secondPod.Status.PodIP}, rc, pingPassedWindows) +// } + +// func WindowsPodToNode(ctx context.Context, clientset *kubernetes.Clientset, nodeName, nodeIP, podNamespace, labelSelector string, rc *restclient.Config) error { +// logrus.Infof("Get Pods by Node: %s %s", nodeName, nodeIP) +// pods, err := k8sutils.GetPodsByNode(ctx, clientset, podNamespace, labelSelector, nodeName) +// if err != nil { +// logrus.Error(err) +// return errors.Wrap(err, "k8s api call") +// } +// if len(pods.Items) <= 1 { +// return errors.New("Less than 2 pods on node") +// } +// // Get first pod on this node +// firstPod, err := clientset.CoreV1().Pods(podNamespace).Get(ctx, pods.Items[0].Name, metav1.GetOptions{}) +// if err != nil { +// return errors.Wrap(err, fmt.Sprintf("Getting pod %s failed with %v", firstPod.Name, err)) +// } +// logrus.Infof("First pod: %v", firstPod.Name) + +// // Get the second pod on this node +// secondPod, err := clientset.CoreV1().Pods(podNamespace).Get(ctx, pods.Items[1].Name, metav1.GetOptions{}) +// if err != nil { +// return errors.Wrap(err, fmt.Sprintf("Getting pod %s failed with %v", secondPod.Name, err)) +// } +// logrus.Infof("Second pod: %v", secondPod.Name) + +// // Ping from pod to node +// resultOne := podTest(ctx, clientset, firstPod, []string{"ping", nodeIP}, rc, pingPassedWindows) +// resultTwo := podTest(ctx, clientset, secondPod, []string{"ping", nodeIP}, rc, pingPassedWindows) + +// if resultOne != nil { +// return resultOne +// } + +// if resultTwo != nil { +// return resultTwo +// } + +// return nil +// } + +// func WindowsPodToInternet(ctx context.Context, clientset *kubernetes.Clientset, nodeName, podNamespace, labelSelector string, rc *restclient.Config) error { +// logrus.Infof("Get Pods by Node: %s", nodeName) +// pods, err := k8sutils.GetPodsByNode(ctx, clientset, podNamespace, labelSelector, nodeName) +// if err != nil { +// logrus.Error(err) +// return errors.Wrap(err, "k8s api call") +// } +// if len(pods.Items) <= 1 { +// return errors.New("Less than 2 pods on node") +// } + +// // Get first pod on this node +// firstPod, err := clientset.CoreV1().Pods(podNamespace).Get(ctx, pods.Items[0].Name, metav1.GetOptions{}) +// if err != nil { +// return errors.Wrap(err, fmt.Sprintf("Getting pod %s failed with %v", firstPod.Name, err)) +// } +// logrus.Infof("First pod: %v", firstPod.Name) + +// // Get the second pod on this node +// secondPod, err := clientset.CoreV1().Pods(podNamespace).Get(ctx, pods.Items[1].Name, metav1.GetOptions{}) +// if err != nil { +// return errors.Wrap(err, fmt.Sprintf("Getting pod %s failed with %v", secondPod.Name, err)) +// } +// logrus.Infof("Second pod: %v", secondPod.Name) + +// resultOne := podTest(ctx, clientset, firstPod, []string{"powershell", "Invoke-WebRequest", "www.bing.com", "-UseBasicParsing"}, rc, webRequestPassedWindows) +// resultTwo := podTest(ctx, clientset, secondPod, []string{"powershell", "Invoke-WebRequest", "www.bing.com", "-UseBasicParsing"}, rc, webRequestPassedWindows) + +// if resultOne != nil { +// return resultOne +// } + +// if resultTwo != nil { +// return resultTwo +// } + +// return nil +// } + +// func webRequestPassedWindows(output string) error { +// const searchString = "200 OK" +// if strings.Contains(output, searchString) { +// return nil +// } +// return errors.Wrapf(errors.New("Output did not contain \"200 OK\""), "output was: %s", output) +// } + +// func pingPassedWindows(output string) error { +// const searchString = "0% loss" +// if strings.Contains(output, searchString) { +// return nil +// } +// return errors.Wrapf(errors.New("Ping did not contain\"0% loss\""), "output was: %s", output) +// } diff --git a/test/internal/k8sutils/utils.go b/test/internal/k8sutils/utils.go index 5b208d153a..66350062b9 100644 --- a/test/internal/k8sutils/utils.go +++ b/test/internal/k8sutils/utils.go @@ -71,7 +71,6 @@ func mustParseResource(path string, out interface{}) error { if err := yaml.NewYAMLOrJSONDecoder(f, 0).Decode(out); err != nil { return err } - return err } From 54e90e36bf4bee6853cc073cf28ac9fb33e425c9 Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Fri, 23 Jun 2023 21:22:21 -0400 Subject: [PATCH 02/45] datapath linux acn pipeline --- .../datapath/datapath_linux_test.go | 145 ++++++++++++++---- test/integration/goldpinger/client.go | 1 + .../datapath/linux-deployment-ipv6.yaml | 88 +++++++++++ .../manifests/datapath/linux-deployment.yaml | 52 ++++--- .../goldpinger/cluster-role-binding.yaml | 2 +- .../manifests/goldpinger/daemonset-ipv6.yaml | 82 ++++++++++ 6 files changed, 320 insertions(+), 50 deletions(-) create mode 100644 test/integration/manifests/datapath/linux-deployment-ipv6.yaml create mode 100644 test/integration/manifests/goldpinger/daemonset-ipv6.yaml diff --git a/test/integration/datapath/datapath_linux_test.go b/test/integration/datapath/datapath_linux_test.go index acf9cea2f1..a0a4ab9a38 100644 --- a/test/integration/datapath/datapath_linux_test.go +++ b/test/integration/datapath/datapath_linux_test.go @@ -6,6 +6,8 @@ import ( "context" "flag" "fmt" + "net" + "os" "testing" "time" @@ -16,13 +18,15 @@ import ( "github.com/pkg/errors" "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" apiv1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) const ( - LinuxDeployYamlPath = "../manifests/datapath/linux-deployment.yaml" + LinuxDeployIPV4 = "../manifests/datapath/linux-deployment.yaml" + LinuxDeployIPv6 = "../manifests/datapath/linux-deployment-ipv6.yaml" podLabelKey = "app" podCount = 2 nodepoolKey = "agentpool" @@ -31,12 +35,20 @@ const ( defaultRetryDelaySeconds = 1 goldpingerRetryCount = 24 goldpingerDelayTimeSeconds = 5 + gpFolder = "../manifests/goldpinger" + gpClusterRolePath = gpFolder + "/cluster-role.yaml" + gpClusterRoleBindingPath = gpFolder + "/cluster-role-binding.yaml" + gpServiceAccountPath = gpFolder + "/service-account.yaml" + gpDaemonset = gpFolder + "/daemonset.yaml" + gpDaemonsetIPv6 = gpFolder + "/daemonset-ipv6.yaml" + gpDeployment = gpFolder + "/deployment.yaml" ) var ( podPrefix = flag.String("podName", "goldpinger", "Prefix for test pods") - podNamespace = flag.String("namespace", "datapath-linux", "Namespace for test pods") + podNamespace = flag.String("namespace", "default", "Namespace for test pods") nodepoolSelector = flag.String("nodepoolSelector", "nodepool1", "Provides nodepool as a Node-Selector for pods") + testProfile = flag.String("testName", LinuxDeployIPV4, "Linux datapath test profile") defaultRetrier = retry.Retrier{ Attempts: 10, Delay: defaultRetryDelaySeconds * time.Second, @@ -50,18 +62,23 @@ k8s cluster with a Linux nodepool consisting of at least 2 Linux nodes. -nodepoolSelector="yournodepoolname" To run the test use one of the following commands: -go test -count=1 test/integration/datapath/datapath_linux_test.go -timeout 3m -tags connection -run ^TestDatapathLinux$ -tags=connection +go test -count=1 test/integration/datapath/datapath_linux_test.go -timeout 3m -tags connection -run ^TestDatapathLinux$ -tags=connection,integration or -go test -count=1 test/integration/datapath/datapath_linux_test.go -timeout 3m -tags connection -run ^TestDatapathLinux$ -podName=acnpod -nodepoolSelector=npwina -tags=connection +go test -count=1 test/integration/datapath/datapath_linux_test.go -timeout 3m -tags connection -run ^TestDatapathLinux$ -podName=acnpod -nodepoolSelector=aks-pool1 -tags=connection,integration -This test checks pod to pod, pod to node, and pod to internet for datapath connectivity. +This test checks pod to pod, pod to node, pod to Internet check Timeout context is controled by the -timeout flag. */ -func TestDatapathLinux(t *testing.T) { +// return podLabelSelector and nodeLabelSelector +func createLabelSelectors() (string, string) { + return fmt.Sprintf("%s=%s", podLabelKey, *podPrefix), fmt.Sprintf("%s=%s", nodepoolKey, *nodepoolSelector) +} + +func setupLinuxEnvironment(t *testing.T) { ctx := context.Background() t.Log("Create Clientset") @@ -69,13 +86,9 @@ func TestDatapathLinux(t *testing.T) { if err != nil { require.NoError(t, err, "could not get k8s clientset: %v", err) } - t.Log("Get REST config") - restConfig := k8sutils.MustGetRestConfig(t) t.Log("Create Label Selectors") - - podLabelSelector := fmt.Sprintf("%s=%s", podLabelKey, *podPrefix) - nodeLabelSelector := fmt.Sprintf("%s=%s", nodepoolKey, *nodepoolSelector) + podLabelSelector, nodeLabelSelector := createLabelSelectors() t.Log("Get Nodes") nodes, err := k8sutils.GetNodeListByLabelSelector(ctx, clientset, nodeLabelSelector) @@ -83,19 +96,35 @@ func TestDatapathLinux(t *testing.T) { require.NoError(t, err, "could not get k8s node list: %v", err) } - // Test Namespace - t.Log("Create Namespace") - err = k8sutils.MustCreateNamespace(ctx, clientset, *podNamespace) createPodFlag := !(apierrors.IsAlreadyExists(err)) t.Logf("%v", createPodFlag) if createPodFlag { + var daemonset appsv1.DaemonSet t.Log("Creating Linux pods through deployment") - deployment, err := k8sutils.MustParseDeployment(LinuxDeployYamlPath) + deployment, err := k8sutils.MustParseDeployment(*testProfile) if err != nil { require.NoError(t, err) } + if *testProfile == LinuxDeployIPV4 { + daemonset, err = k8sutils.MustParseDaemonSet(gpDaemonset) + if err != nil { + t.Fatal(err) + } + } else { + daemonset, err = k8sutils.MustParseDaemonSet(gpDaemonsetIPv6) + if err != nil { + t.Fatal(err) + } + } + + rbacCleanUpFn, err := k8sutils.MustSetUpClusterRBAC(ctx, clientset, gpClusterRolePath, gpClusterRoleBindingPath, gpServiceAccountPath) + if err != nil { + t.Log(os.Getwd()) + t.Fatal(err) + } + // Fields for overwritting existing deployment yaml. // Defaults from flags will not change anything deployment.Spec.Selector.MatchLabels[podLabelKey] = *podPrefix @@ -104,14 +133,30 @@ func TestDatapathLinux(t *testing.T) { deployment.Name = *podPrefix deployment.Namespace = *podNamespace - t.Logf("deployment Spec Template is %+v", deployment.Spec.Template) deploymentsClient := clientset.AppsV1().Deployments(*podNamespace) err = k8sutils.MustCreateDeployment(ctx, deploymentsClient, deployment) if err != nil { require.NoError(t, err) } - t.Logf("podNamespace is %s", *podNamespace) - t.Logf("podLabelSelector is %s", podLabelSelector) + + daemonsetClient := clientset.AppsV1().DaemonSets(daemonset.Namespace) + err = k8sutils.MustCreateDaemonset(ctx, daemonsetClient, daemonset) + if err != nil { + t.Fatal(err) + } + + t.Cleanup(func() { + t.Log("cleaning up resources") + rbacCleanUpFn() + + if err := deploymentsClient.Delete(ctx, deployment.Name, metav1.DeleteOptions{}); err != nil { + t.Log(err) + } + + if err := daemonsetClient.Delete(ctx, daemonset.Name, metav1.DeleteOptions{}); err != nil { + t.Log(err) + } + }) t.Log("Waiting for pods to be running state") err = k8sutils.WaitForPodsRunning(ctx, clientset, *podNamespace, podLabelSelector) @@ -120,18 +165,15 @@ func TestDatapathLinux(t *testing.T) { } t.Log("Successfully created customer linux pods") } else { - // Checks namespace already exists from previous attempt - t.Log("Namespace already exists") - t.Log("Checking for pods to be running state") err = k8sutils.WaitForPodsRunning(ctx, clientset, *podNamespace, podLabelSelector) if err != nil { require.NoError(t, err) } } + t.Log("Checking Linux test environment") for _, node := range nodes.Items { - pods, err := k8sutils.GetPodsByNode(ctx, clientset, *podNamespace, podLabelSelector, node.Name) if err != nil { require.NoError(t, err, "could not get k8s clientset: %v", err) @@ -140,8 +182,22 @@ func TestDatapathLinux(t *testing.T) { t.Logf("%s", node.Name) require.NoError(t, errors.New("Less than 2 pods on node")) } + } t.Log("Linux test environment ready") +} + +func TestDatapathLinux(t *testing.T) { + ctx := context.Background() + + t.Log("Get REST config") + restConfig := k8sutils.MustGetRestConfig(t) + + t.Log("Create Clientset") + clientset, _ := k8sutils.MustGetClientset() + + setupLinuxEnvironment(t) + podLabelSelector, _ := createLabelSelectors() t.Run("Linux ping tests", func(t *testing.T) { // Check goldpinger health @@ -150,7 +206,6 @@ func TestDatapathLinux(t *testing.T) { checkPodIPsFn := func() error { podList, err := podsClient.List(ctx, metav1.ListOptions{LabelSelector: "app=goldpinger"}) - t.Logf("podList is %+v", podList) if err != nil { return err } @@ -180,10 +235,46 @@ func TestDatapathLinux(t *testing.T) { t.Log("all pods have been allocated IPs") }) + // TODO: avoid using yaml file path to control test case + if *testProfile == LinuxDeployIPv6 { + t.Run("Linux dualstack overlay tests", func(t *testing.T) { + t.Run("test dualstack overlay", func(t *testing.T) { + podsClient := clientset.CoreV1().Pods(*podNamespace) + + checkPodIPsFn := func() error { + podList, err := podsClient.List(ctx, metav1.ListOptions{LabelSelector: "app=goldpinger"}) + if err != nil { + return err + } + + for _, pod := range podList.Items { + podIPs := pod.Status.PodIPs + if len(podIPs) < 2 { + return errors.New("a pod only gets one IP") + } + if net.ParseIP(podIPs[0].IP).To4() == nil || net.ParseIP(podIPs[1].IP).To16() == nil { + return errors.New("a pod does not have both ipv4 and ipv6 address") + } + } + return nil + } + err := defaultRetrier.Do(ctx, checkPodIPsFn) + if err != nil { + t.Fatalf("dualstack overlay pod properties check is failed due to: %v", err) + } + + t.Log("all dualstack linux pods properties have been verified") + }) + }) + } + t.Run("all linux pods can ping each other", func(t *testing.T) { + clusterCheckCtx, cancel := context.WithTimeout(ctx, 3*time.Minute) + defer cancel() + pfOpts := k8s.PortForwardingOpts{ - Namespace: "default", - LabelSelector: "type=goldpinger-pod", + Namespace: *podNamespace, + LabelSelector: podLabelSelector, LocalPort: 9090, DestPort: 8080, } @@ -210,15 +301,11 @@ func TestDatapathLinux(t *testing.T) { defer pf.Stop() gpClient := goldpinger.Client{Host: pf.Address()} - - clusterCheckCtx, cancel := context.WithTimeout(ctx, 3*time.Minute) - defer cancel() clusterCheckFn := func() error { clusterState, err := gpClient.CheckAll(clusterCheckCtx) if err != nil { return err } - stats := goldpinger.ClusterStats(clusterState) stats.PrintStats() if stats.AllPingsHealthy() { diff --git a/test/integration/goldpinger/client.go b/test/integration/goldpinger/client.go index 49b29d9686..dac4149ced 100644 --- a/test/integration/goldpinger/client.go +++ b/test/integration/goldpinger/client.go @@ -1,3 +1,4 @@ +//go:build integration // +build integration package goldpinger diff --git a/test/integration/manifests/datapath/linux-deployment-ipv6.yaml b/test/integration/manifests/datapath/linux-deployment-ipv6.yaml new file mode 100644 index 0000000000..32a5ef626a --- /dev/null +++ b/test/integration/manifests/datapath/linux-deployment-ipv6.yaml @@ -0,0 +1,88 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: goldpinger-deploy + namespace: default +spec: + replicas: 4 + selector: + matchLabels: + app: goldpinger + template: + metadata: + labels: + app: goldpinger + spec: + containers: + - name: goldpinger + env: + - name: HOST + value: "0.0.0.0" + - name: PORT + value: "8080" + - name: PING_TIMEOUT + value: "10s" + - name: CHECK_TIMEOUT + value: "20s" + - name: CHECK_ALL_TIMEOUT + value: "20s" + - name: DNS_TARGETS_TIMEOUT + value: "10s" + - name: IP_VERSIONS + value: "6" + - name: HOSTNAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: HOSTS_TO_RESOLVE + value: "2001:4860:4860::8888 www.bing.com" + image: "docker.io/bloomberg/goldpinger:v3.7.0" + serviceAccount: goldpinger-serviceaccount + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + securityContext: + allowPrivilegeEscalation: false + securityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 2000 + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: "app" + operator: In + values: + - goldpinger + topologyKey: "kubernetes.io/hostname" + resources: + limits: + memory: 80Mi + requests: + cpu: 1m + memory: 40Mi + ports: + - containerPort: 8080 + name: http + readinessProbe: + httpGet: + path: /healthz + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 5 + livenessProbe: + httpGet: + path: /healthz + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 5 + nodeSelector: + kubernetes.io/os: linux diff --git a/test/integration/manifests/datapath/linux-deployment.yaml b/test/integration/manifests/datapath/linux-deployment.yaml index 4b029c28c3..24ddb5f8c5 100644 --- a/test/integration/manifests/datapath/linux-deployment.yaml +++ b/test/integration/manifests/datapath/linux-deployment.yaml @@ -2,7 +2,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: goldpinger-deploy - namespace: datapath-linux + namespace: default spec: replicas: 4 selector: @@ -13,23 +13,6 @@ spec: labels: app: goldpinger spec: - serviceAccount: "goldpinger-serviceaccount" - securityContext: - runAsNonRoot: true - runAsUser: 1000 - fsGroup: 2000 - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: "app" - operator: In - values: - - goldpinger - topologyKey: "kubernetes.io/hostname" containers: - name: goldpinger env: @@ -37,6 +20,14 @@ spec: value: "0.0.0.0" - name: PORT value: "8080" + - name: PING_TIMEOUT + value: "10s" + - name: CHECK_TIMEOUT + value: "20s" + - name: CHECK_ALL_TIMEOUT + value: "20s" + - name: DNS_TARGETS_TIMEOUT + value: "10s" - name: HOSTNAME valueFrom: fieldRef: @@ -45,10 +36,31 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - image: "docker.io/bloomberg/goldpinger:v3.0.0" + - name: HOSTS_TO_RESOLVE + value: "1.1.1.1 8.8.8.8 www.bing.com" + image: "docker.io/bloomberg/goldpinger:v3.7.0" + serviceAccount: goldpinger-serviceaccount + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule securityContext: allowPrivilegeEscalation: false - readOnlyRootFilesystem: true + securityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 2000 + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: "app" + operator: In + values: + - goldpinger + topologyKey: "kubernetes.io/hostname" resources: limits: memory: 80Mi diff --git a/test/integration/manifests/goldpinger/cluster-role-binding.yaml b/test/integration/manifests/goldpinger/cluster-role-binding.yaml index c7c22e9bb3..e18b186a12 100644 --- a/test/integration/manifests/goldpinger/cluster-role-binding.yaml +++ b/test/integration/manifests/goldpinger/cluster-role-binding.yaml @@ -1,4 +1,4 @@ -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: goldpinger-clusterrolebinding diff --git a/test/integration/manifests/goldpinger/daemonset-ipv6.yaml b/test/integration/manifests/goldpinger/daemonset-ipv6.yaml new file mode 100644 index 0000000000..77dfe0bf79 --- /dev/null +++ b/test/integration/manifests/goldpinger/daemonset-ipv6.yaml @@ -0,0 +1,82 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: goldpinger-host + namespace: default +spec: + selector: + matchLabels: + app: goldpinger + type: goldpinger-host + template: + metadata: + labels: + app: goldpinger + type: goldpinger-host + spec: + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + hostNetwork: true + serviceAccount: "goldpinger-serviceaccount" + securityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 2000 + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: "app" + operator: In + values: + - goldpinger + topologyKey: "kubernetes.io/hostname" + containers: + - name: goldpinger-vm + env: + - name: HOST + value: "0.0.0.0" + - name: PORT + value: "8080" + - name: PING_TIMEOUT + value: "10s" + - name: CHECK_TIMEOUT + value: "20s" + - name: CHECK_ALL_TIMEOUT + value: "20s" + - name: IP_VERSIONS + value: "6" + - name: HOSTNAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP +# - name: HOSTS_TO_RESOLVE +# value: "1.1.1.1 8.8.8.8 www.bing.com" + image: "docker.io/bloomberg/goldpinger:v3.7.0" + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + ports: + - containerPort: 8080 + name: http + readinessProbe: + httpGet: + path: /healthz + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 5 + livenessProbe: + httpGet: + path: /healthz + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 5 From 044f9e92c203e355430916a4f0656183406b4f82 Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Fri, 23 Jun 2023 21:23:37 -0400 Subject: [PATCH 03/45] acn datapath linux --- test/internal/datapath/datapath_linux.go | 166 ----------------------- 1 file changed, 166 deletions(-) diff --git a/test/internal/datapath/datapath_linux.go b/test/internal/datapath/datapath_linux.go index f32d4cda3f..d4db62b15d 100644 --- a/test/internal/datapath/datapath_linux.go +++ b/test/internal/datapath/datapath_linux.go @@ -1,167 +1 @@ package datapath - -import ( - "context" - "fmt" - - "github.com/Azure/azure-container-networking/test/internal/k8sutils" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - restclient "k8s.io/client-go/rest" -) - -func LinuxPodToPodPingTestSameNode(ctx context.Context, clientset *kubernetes.Clientset, nodeName, podNamespace, labelSelector string, rc *restclient.Config) error { - logrus.Infof("Get Pods for Linux Node: %s", nodeName) - pods, err := k8sutils.GetPodsByNode(ctx, clientset, podNamespace, labelSelector, nodeName) - if err != nil { - logrus.Error(err) - return errors.Wrap(err, "k8s api call") - } - if len(pods.Items) <= 1 { - return errors.New("Less than 2 pods on node") - } - - // Get first pod on this node - firstPod, err := clientset.CoreV1().Pods(podNamespace).Get(ctx, pods.Items[0].Name, metav1.GetOptions{}) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("Getting pod %s failed with %v", firstPod.Name, err)) - } - logrus.Infof("First pod: %v %v", firstPod.Name, firstPod.Status.PodIP) - - // Get the second pod on this node - secondPod, err := clientset.CoreV1().Pods(podNamespace).Get(ctx, pods.Items[1].Name, metav1.GetOptions{}) - if err != nil { - return errors.Wrap(err, fmt.Sprintf("Getting pod %s failed with %v", secondPod.Name, err)) - } - logrus.Infof("Second pod: %v %v", secondPod.Name, secondPod.Status.PodIP) - - // Ping the second pod from the first pod - return podTest(ctx, clientset, firstPod, []string{"ping", secondPod.Status.PodIP}, rc, pingPassedWindows) -} - -// func WindowsPodToPodPingTestDiffNode(ctx context.Context, clientset *kubernetes.Clientset, nodeName1, nodeName2, podNamespace, labelSelector string, rc *restclient.Config) error { -// logrus.Infof("Get Pods for Node 1: %s", nodeName1) -// // Node 1 -// pods, err := k8sutils.GetPodsByNode(ctx, clientset, podNamespace, labelSelector, nodeName1) -// if err != nil { -// logrus.Error(err) -// return errors.Wrap(err, "k8s api call") -// } -// firstPod, err := clientset.CoreV1().Pods(podNamespace).Get(ctx, pods.Items[0].Name, metav1.GetOptions{}) -// if err != nil { -// return errors.Wrap(err, fmt.Sprintf("Getting pod %s failed with %v", firstPod.Name, err)) -// } -// logrus.Infof("First pod: %v %v", firstPod.Name, firstPod.Status.PodIP) - -// logrus.Infof("Get Pods for Node 2: %s", nodeName2) -// // Node 2 -// pods, err = k8sutils.GetPodsByNode(ctx, clientset, podNamespace, labelSelector, nodeName2) -// if err != nil { -// logrus.Error(err) -// return errors.Wrap(err, "k8s api call") -// } -// secondPod, err := clientset.CoreV1().Pods(podNamespace).Get(ctx, pods.Items[0].Name, metav1.GetOptions{}) -// if err != nil { -// return errors.Wrap(err, fmt.Sprintf("Getting pod %s failed with %v", secondPod.Name, err)) -// } -// logrus.Infof("Second pod: %v %v", secondPod.Name, secondPod.Status.PodIP) - -// // Ping the second pod from the first pod located on different nodes -// return podTest(ctx, clientset, firstPod, []string{"ping", secondPod.Status.PodIP}, rc, pingPassedWindows) -// } - -// func WindowsPodToNode(ctx context.Context, clientset *kubernetes.Clientset, nodeName, nodeIP, podNamespace, labelSelector string, rc *restclient.Config) error { -// logrus.Infof("Get Pods by Node: %s %s", nodeName, nodeIP) -// pods, err := k8sutils.GetPodsByNode(ctx, clientset, podNamespace, labelSelector, nodeName) -// if err != nil { -// logrus.Error(err) -// return errors.Wrap(err, "k8s api call") -// } -// if len(pods.Items) <= 1 { -// return errors.New("Less than 2 pods on node") -// } -// // Get first pod on this node -// firstPod, err := clientset.CoreV1().Pods(podNamespace).Get(ctx, pods.Items[0].Name, metav1.GetOptions{}) -// if err != nil { -// return errors.Wrap(err, fmt.Sprintf("Getting pod %s failed with %v", firstPod.Name, err)) -// } -// logrus.Infof("First pod: %v", firstPod.Name) - -// // Get the second pod on this node -// secondPod, err := clientset.CoreV1().Pods(podNamespace).Get(ctx, pods.Items[1].Name, metav1.GetOptions{}) -// if err != nil { -// return errors.Wrap(err, fmt.Sprintf("Getting pod %s failed with %v", secondPod.Name, err)) -// } -// logrus.Infof("Second pod: %v", secondPod.Name) - -// // Ping from pod to node -// resultOne := podTest(ctx, clientset, firstPod, []string{"ping", nodeIP}, rc, pingPassedWindows) -// resultTwo := podTest(ctx, clientset, secondPod, []string{"ping", nodeIP}, rc, pingPassedWindows) - -// if resultOne != nil { -// return resultOne -// } - -// if resultTwo != nil { -// return resultTwo -// } - -// return nil -// } - -// func WindowsPodToInternet(ctx context.Context, clientset *kubernetes.Clientset, nodeName, podNamespace, labelSelector string, rc *restclient.Config) error { -// logrus.Infof("Get Pods by Node: %s", nodeName) -// pods, err := k8sutils.GetPodsByNode(ctx, clientset, podNamespace, labelSelector, nodeName) -// if err != nil { -// logrus.Error(err) -// return errors.Wrap(err, "k8s api call") -// } -// if len(pods.Items) <= 1 { -// return errors.New("Less than 2 pods on node") -// } - -// // Get first pod on this node -// firstPod, err := clientset.CoreV1().Pods(podNamespace).Get(ctx, pods.Items[0].Name, metav1.GetOptions{}) -// if err != nil { -// return errors.Wrap(err, fmt.Sprintf("Getting pod %s failed with %v", firstPod.Name, err)) -// } -// logrus.Infof("First pod: %v", firstPod.Name) - -// // Get the second pod on this node -// secondPod, err := clientset.CoreV1().Pods(podNamespace).Get(ctx, pods.Items[1].Name, metav1.GetOptions{}) -// if err != nil { -// return errors.Wrap(err, fmt.Sprintf("Getting pod %s failed with %v", secondPod.Name, err)) -// } -// logrus.Infof("Second pod: %v", secondPod.Name) - -// resultOne := podTest(ctx, clientset, firstPod, []string{"powershell", "Invoke-WebRequest", "www.bing.com", "-UseBasicParsing"}, rc, webRequestPassedWindows) -// resultTwo := podTest(ctx, clientset, secondPod, []string{"powershell", "Invoke-WebRequest", "www.bing.com", "-UseBasicParsing"}, rc, webRequestPassedWindows) - -// if resultOne != nil { -// return resultOne -// } - -// if resultTwo != nil { -// return resultTwo -// } - -// return nil -// } - -// func webRequestPassedWindows(output string) error { -// const searchString = "200 OK" -// if strings.Contains(output, searchString) { -// return nil -// } -// return errors.Wrapf(errors.New("Output did not contain \"200 OK\""), "output was: %s", output) -// } - -// func pingPassedWindows(output string) error { -// const searchString = "0% loss" -// if strings.Contains(output, searchString) { -// return nil -// } -// return errors.Wrapf(errors.New("Ping did not contain\"0% loss\""), "output was: %s", output) -// } From 1e75bda2645f3153b1b62bb7fc5f38fa57159b93 Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Fri, 23 Jun 2023 21:50:14 -0400 Subject: [PATCH 04/45] add linux acn control plane test cases --- test/integration/load/load_test.go | 21 +++ .../manifests/load/noop-deployment.yaml | 2 + .../manifests/load/privileged-daemonset.yaml | 7 + test/validate/client.go | 2 + test/validate/linux_validate.go | 157 ++++++++++++++++-- 5 files changed, 174 insertions(+), 15 deletions(-) diff --git a/test/integration/load/load_test.go b/test/integration/load/load_test.go index dbceb15ed9..4e1b33bb2d 100644 --- a/test/integration/load/load_test.go +++ b/test/integration/load/load_test.go @@ -29,6 +29,7 @@ var ( skipWait = flag.Bool("skip-wait", false, "Skip waiting for pods to be ready") restartCase = flag.Bool("restart-case", false, "In restart case, skip if we don't find state file") namespace = "load-test" + validateDualStack = flag.Bool("validate-dualstack", false, "Validate the dualstack overlay") ) /* @@ -176,3 +177,23 @@ func TestScaleDeployment(t *testing.T) { t.Fatal(err) } } + +func TestDualStackProperties(t *testing.T) { + clientset, err := k8sutils.MustGetClientset() + if err != nil { + t.Fatal(err) + } + config := k8sutils.MustGetRestConfig(t) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) + defer cancel() + + t.Log("Validating the dualstack node labels") + validatorClient := validate.GetValidatorClient(*osType) + validator := validatorClient.CreateClient(ctx, clientset, config, namespace, *cniType, *restartCase) + + // validate dualstack overlay scenarios + err = validator.ValidateDualStackNodeProperties() + if err != nil { + t.Fatal(err) + } +} diff --git a/test/integration/manifests/load/noop-deployment.yaml b/test/integration/manifests/load/noop-deployment.yaml index 85272941c6..276d6f215f 100644 --- a/test/integration/manifests/load/noop-deployment.yaml +++ b/test/integration/manifests/load/noop-deployment.yaml @@ -19,3 +19,5 @@ spec: imagePullPolicy: Always securityContext: privileged: true + nodeSelector: + kubernetes.io/os: linux \ No newline at end of file diff --git a/test/integration/manifests/load/privileged-daemonset.yaml b/test/integration/manifests/load/privileged-daemonset.yaml index 9bacdc4ebe..4a81e6fa81 100644 --- a/test/integration/manifests/load/privileged-daemonset.yaml +++ b/test/integration/manifests/load/privileged-daemonset.yaml @@ -28,11 +28,18 @@ spec: name: azure-cns - mountPath: /host name: host-root + - mountPath: /var/run + name: azure-cns-noncilium volumes: - name: azure-cns hostPath: path: /var/run/azure-cns + - name: azure-cns-noncilium + hostPath: + path: /var/run - hostPath: path: / type: "" name: host-root + nodeSelector: + kubernetes.io/os: linux diff --git a/test/validate/client.go b/test/validate/client.go index bc6e0aeb54..11b52081e5 100644 --- a/test/validate/client.go +++ b/test/validate/client.go @@ -20,6 +20,8 @@ type Validator struct { type IValidator interface { ValidateStateFile() error ValidateRestartNetwork() error + ValidateDualStackNodeProperties() error + // ValidateDataPath() error } diff --git a/test/validate/linux_validate.go b/test/validate/linux_validate.go index d2839f4098..dca3712c98 100644 --- a/test/validate/linux_validate.go +++ b/test/validate/linux_validate.go @@ -9,6 +9,7 @@ import ( restserver "github.com/Azure/azure-container-networking/cns/restserver" k8sutils "github.com/Azure/azure-container-networking/test/internal/k8sutils" "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" ) @@ -18,17 +19,24 @@ const ( privilegedLabelSelector = "app=privileged-daemonset" privilegedNamespace = "kube-system" - cnsLabelSelector = "k8s-app=azure-cns" - ciliumLabelSelector = "k8s-app=cilium" + cnsLabelSelector = "k8s-app=azure-cns" + ciliumLabelSelector = "k8s-app=cilium" + overlayClusterLabelName = "overlay" ) var ( - restartNetworkCmd = []string{"bash", "-c", "chroot /host /bin/bash -c 'systemctl restart systemd-networkd'"} - cnsStateFileCmd = []string{"bash", "-c", "cat /var/run/azure-cns/azure-endpoints.json"} - ciliumStateFileCmd = []string{"bash", "-c", "cilium endpoint list -o json"} - cnsLocalCacheCmd = []string{"curl", "localhost:10090/debug/ipaddresses", "-d", "{\"IPConfigStateFilter\":[\"Assigned\"]}"} + restartNetworkCmd = []string{"bash", "-c", "chroot /host /bin/bash -c 'systemctl restart systemd-networkd'"} + cnsStateFileCmd = []string{"bash", "-c", "cat /var/run/azure-cns/azure-endpoints.json"} + azureCnsStateFileCmd = []string{"bash", "-c", "cat /var/run/azure-vnet.json"} // azure cni statefile is /var/run/azure-vnet.json + ciliumStateFileCmd = []string{"bash", "-c", "cilium endpoint list -o json"} + cnsLocalCacheCmd = []string{"curl", "localhost:10090/debug/ipaddresses", "-d", "{\"IPConfigStateFilter\":[\"Assigned\"]}"} ) +var dualstackoverlaynodelabel = map[string]string{ + "kubernetes.azure.com/podnetwork-type": "overlay", + "kubernetes.azure.com/podv6network-type": "overlay", +} + type stateFileIpsFunc func([]byte) (map[string]string, error) type LinuxClient struct{} @@ -62,6 +70,63 @@ type Address struct { Addr string `json:"ipv4"` } +// parse azure-vnet.json +// azure cni manages endpoint state +type AzureCniState struct { + AzureCniState Network `json:"Network"` +} + +type Network struct { + Version string `json:"Version"` + TimeStamp string `json:"TimeStamp"` + ExternalInterfaces map[string]InterfaceInfo `json:"ExternalInterfaces"` // key: interface name; value: Interface Info +} + +type InterfaceInfo struct { + Name string `json:"Name"` + Networks map[string]NetworkInfo `json:"Networks"` // key: networkName, value: NetworkInfo +} + +type AzureVnetInfo struct { + Name string + Networks map[string]NetworkInfo // key: network name, value: NetworkInfo +} + +type NetworkInfo struct { + ID string + Mode string + Subnets []Subnet + Endpoints map[string]AzureVnetEndpointInfo // key: azure endpoint name, value: AzureVnetEndpointInfo + PODName string +} + +type Subnet struct { + Family int + Prefix Prefix + Gateway string + PrimaryIP string +} + +type Prefix struct { + IP string + Mask string +} + +type AzureVnetEndpointInfo struct { + IfName string + MacAddress string + IPAddresses []Prefix + PODName string +} + +type check struct { + name string + stateFileIps func([]byte) (map[string]string, error) + podLabelSelector string + podNamespace string + cmd []string +} + func (l *LinuxClient) CreateClient(ctx context.Context, clienset *kubernetes.Clientset, config *rest.Config, namespace, cni string, restartCase bool) IValidator { // deploy privileged pod privilegedDaemonSet, err := k8sutils.MustParseDaemonSet(privilegedDaemonSetPath) @@ -89,21 +154,22 @@ func (l *LinuxClient) CreateClient(ctx context.Context, clienset *kubernetes.Cli } } -// Todo: Based on cni version validate different state files func (v *LinuxValidator) ValidateStateFile() error { - checks := []struct { - name string - stateFileIps func([]byte) (map[string]string, error) - podLabelSelector string - podNamespace string - cmd []string - }{ + checkSet := make(map[string][]check) // key is cni type, value is a list of check + + // TODO: add cniv1 when adding related test cases + checkSet["cilium"] = []check{ {"cns", cnsStateFileIps, cnsLabelSelector, privilegedNamespace, cnsStateFileCmd}, {"cilium", ciliumStateFileIps, ciliumLabelSelector, privilegedNamespace, ciliumStateFileCmd}, {"cns cache", cnsCacheStateFileIps, cnsLabelSelector, privilegedNamespace, cnsLocalCacheCmd}, } - for _, check := range checks { + checkSet["cniv2"] = []check{ + {"cns cache", cnsCacheStateFileIps, cnsLabelSelector, privilegedNamespace, cnsLocalCacheCmd}, + {"azure cns noncilium", azureCNSNonCiliumStateFileIPs, privilegedLabelSelector, privilegedNamespace, azureCnsStateFileCmd}, + } + + for _, check := range checkSet[v.cni] { err := v.validate(check.stateFileIps, check.cmd, check.name, check.podNamespace, check.podLabelSelector) if err != nil { return err @@ -158,6 +224,31 @@ func cnsStateFileIps(result []byte) (map[string]string, error) { return cnsPodIps, nil } +func azureCNSNonCiliumStateFileIPs(result []byte) (map[string]string, error) { + var azureCNSNonCiliumResult AzureCniState + err := json.Unmarshal(result, &azureCNSNonCiliumResult) + if err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal azure cni endpoint list") + } + + azureCnsPodIps := make(map[string]string) + for _, v := range azureCNSNonCiliumResult.AzureCniState.ExternalInterfaces { + for _, networks := range v.Networks { + for _, ip := range networks.Endpoints { + pod := ip.PODName + ipv4 := ip.IPAddresses[0].IP + azureCnsPodIps[ipv4] = pod + + if len(ip.IPAddresses) > 1 { + ipv6 := ip.IPAddresses[1].IP + azureCnsPodIps[ipv6] = pod + } + } + } + } + return azureCnsPodIps, nil +} + func ciliumStateFileIps(result []byte) (map[string]string, error) { var ciliumResult []CiliumEndpointStatus err := json.Unmarshal(result, &ciliumResult) @@ -230,3 +321,39 @@ func (v *LinuxValidator) validate(stateFileIps stateFileIpsFunc, cmd []string, c log.Printf("State file validation for %s passed", checkType) return nil } + +func (v *LinuxValidator) ValidateDualStackNodeProperties() error { + log.Print("Validating Dualstack Overlay Linux Node properties") + nodes, err := k8sutils.GetNodeList(v.ctx, v.clientset) + if err != nil { + return errors.Wrapf(err, "failed to get node list") + } + + for index := range nodes.Items { + nodeName := nodes.Items[index].ObjectMeta.Name + // check node status + nodeConditions := nodes.Items[index].Status.Conditions + if nodeConditions[len(nodeConditions)-1].Type != corev1.NodeReady { + return errors.Wrapf(err, "node %s status is not ready", nodeName) + } + + // get node labels + nodeLabels := nodes.Items[index].ObjectMeta.GetLabels() + for key := range nodeLabels { + if value, ok := dualstackoverlaynodelabel[key]; ok { + log.Printf("label %s is correctly shown on the node %+v", key, nodeName) + if value != overlayClusterLabelName { + return errors.Wrapf(err, "node %s overlay label name is wrong", nodeName) + } + } + } + + // get node allocated IPs and check whether it includes ipv4 and ipv6 address + // node status addresses object will return three objects; two of them are ip addresses object(one is ipv4 and one is ipv6) + if len(nodes.Items[index].Status.Addresses) < 3 { + return errors.Wrapf(err, "node %s is missing IPv6 internal IP", nodeName) + } + } + + return nil +} From 08eb0486342d1a4b74e6ccd6a8bb3e06b826b7de Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Fri, 23 Jun 2023 21:52:48 -0400 Subject: [PATCH 05/45] add validateDualStack test case --- test/integration/load/load_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/integration/load/load_test.go b/test/integration/load/load_test.go index 4e1b33bb2d..bc49ca23a5 100644 --- a/test/integration/load/load_test.go +++ b/test/integration/load/load_test.go @@ -113,6 +113,10 @@ func TestLoad(t *testing.T) { t.Fatal(err) } + if *validateDualStack { + t.Run("Validate dualstack overlay", TestDualStackProperties) + } + if *validateStateFile { t.Run("Validate state file", TestValidateState) } From 4cc4e7c4331d92e7b4338dac57c11f1b21634f7a Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Fri, 23 Jun 2023 22:08:11 -0400 Subject: [PATCH 06/45] fix issues --- .../manifests/load/privileged-daemonset.yaml | 4 ++-- test/internal/k8sutils/utils_get.go | 6 +++-- test/validate/utils.go | 24 +++++++++++++++++-- 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/test/integration/manifests/load/privileged-daemonset.yaml b/test/integration/manifests/load/privileged-daemonset.yaml index 4a81e6fa81..519c81d3f2 100644 --- a/test/integration/manifests/load/privileged-daemonset.yaml +++ b/test/integration/manifests/load/privileged-daemonset.yaml @@ -24,7 +24,7 @@ spec: privileged: true runAsUser: 0 volumeMounts: - - mountPath: /var/run/azure-cns + - mountPath: /var/run/azure-network name: azure-cns - mountPath: /host name: host-root @@ -33,7 +33,7 @@ spec: volumes: - name: azure-cns hostPath: - path: /var/run/azure-cns + path: /var/run/azure-network - name: azure-cns-noncilium hostPath: path: /var/run diff --git a/test/internal/k8sutils/utils_get.go b/test/internal/k8sutils/utils_get.go index 531ec38fce..f5d72e7f6f 100644 --- a/test/internal/k8sutils/utils_get.go +++ b/test/internal/k8sutils/utils_get.go @@ -43,9 +43,11 @@ func GetPodsIpsByNode(ctx context.Context, clientset *kubernetes.Clientset, name if err != nil { return nil, err } - ips := make([]string, 0, len(pods.Items)) + ips := make([]string, 0, len(pods.Items)*2) for index := range pods.Items { - ips = append(ips, pods.Items[index].Status.PodIP) + for _, podIP := range pods.Items[index].Status.PodIPs { + ips = append(ips, podIP.IP) + } } return ips, nil } diff --git a/test/validate/utils.go b/test/validate/utils.go index 7180c7bc66..faef068958 100644 --- a/test/validate/utils.go +++ b/test/validate/utils.go @@ -2,6 +2,7 @@ package validate import ( "context" + "reflect" "github.com/Azure/azure-container-networking/test/internal/k8sutils" corev1 "k8s.io/api/core/v1" @@ -29,11 +30,30 @@ func getPodIPsWithoutNodeIP(ctx context.Context, clientset *kubernetes.Clientset if err != nil { return podsIpsWithoutNodeIP } - nodeIP := node.Status.Addresses[0].Address + nodeIPs := make([]string, 0) + for _, address := range node.Status.Addresses { + if address.Type == corev1.NodeInternalIP { + nodeIPs = append(nodeIPs, address.Address) + } + } + for _, podIP := range podIPs { - if podIP != nodeIP { + if contains := contain(podIP, nodeIPs); contains == false { podsIpsWithoutNodeIP = append(podsIpsWithoutNodeIP, podIP) } } return podsIpsWithoutNodeIP } + +func contain(obj interface{}, target interface{}) bool { + targetValue := reflect.ValueOf(target) + switch reflect.TypeOf(target).Kind() { + case reflect.Slice, reflect.Array: + for i := 0; i < targetValue.Len(); i++ { + if targetValue.Index(i).Interface() == obj { + return true + } + } + } + return false +} From 61ccd3277ed11a761dad337d122156cdc7ad3e00 Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Fri, 23 Jun 2023 22:17:01 -0400 Subject: [PATCH 07/45] fix linter issues --- test/internal/k8sutils/utils_get.go | 2 +- test/validate/linux_validate.go | 3 ++- test/validate/utils.go | 6 +++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/test/internal/k8sutils/utils_get.go b/test/internal/k8sutils/utils_get.go index f5d72e7f6f..c6c6b545bf 100644 --- a/test/internal/k8sutils/utils_get.go +++ b/test/internal/k8sutils/utils_get.go @@ -43,7 +43,7 @@ func GetPodsIpsByNode(ctx context.Context, clientset *kubernetes.Clientset, name if err != nil { return nil, err } - ips := make([]string, 0, len(pods.Items)*2) + ips := make([]string, 0, len(pods.Items)*2) // no lint for index := range pods.Items { for _, podIP := range pods.Items[index].Status.PodIPs { ips = append(ips, podIP.IP) diff --git a/test/validate/linux_validate.go b/test/validate/linux_validate.go index dca3712c98..e8d68bd717 100644 --- a/test/validate/linux_validate.go +++ b/test/validate/linux_validate.go @@ -22,6 +22,7 @@ const ( cnsLabelSelector = "k8s-app=azure-cns" ciliumLabelSelector = "k8s-app=cilium" overlayClusterLabelName = "overlay" + dualstackNodeStatusAddr = 3 ) var ( @@ -350,7 +351,7 @@ func (v *LinuxValidator) ValidateDualStackNodeProperties() error { // get node allocated IPs and check whether it includes ipv4 and ipv6 address // node status addresses object will return three objects; two of them are ip addresses object(one is ipv4 and one is ipv6) - if len(nodes.Items[index].Status.Addresses) < 3 { + if len(nodes.Items[index].Status.Addresses) < dualstackNodeStatusAddr { return errors.Wrapf(err, "node %s is missing IPv6 internal IP", nodeName) } } diff --git a/test/validate/utils.go b/test/validate/utils.go index faef068958..3aff776ee2 100644 --- a/test/validate/utils.go +++ b/test/validate/utils.go @@ -38,16 +38,16 @@ func getPodIPsWithoutNodeIP(ctx context.Context, clientset *kubernetes.Clientset } for _, podIP := range podIPs { - if contains := contain(podIP, nodeIPs); contains == false { + if !contain(podIP, nodeIPs) { podsIpsWithoutNodeIP = append(podsIpsWithoutNodeIP, podIP) } } return podsIpsWithoutNodeIP } -func contain(obj interface{}, target interface{}) bool { +func contain(obj, target interface{}) bool { targetValue := reflect.ValueOf(target) - switch reflect.TypeOf(target).Kind() { + switch reflect.TypeOf(target).Kind() { // no lint case reflect.Slice, reflect.Array: for i := 0; i < targetValue.Len(); i++ { if targetValue.Index(i).Interface() == obj { From ed5b10be2ca402ab571f0673329d061dec15b84b Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Fri, 23 Jun 2023 22:19:58 -0400 Subject: [PATCH 08/45] fix linter issues --- test/internal/k8sutils/utils_get.go | 2 +- test/validate/utils.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/internal/k8sutils/utils_get.go b/test/internal/k8sutils/utils_get.go index c6c6b545bf..6c1ff2b0e6 100644 --- a/test/internal/k8sutils/utils_get.go +++ b/test/internal/k8sutils/utils_get.go @@ -43,7 +43,7 @@ func GetPodsIpsByNode(ctx context.Context, clientset *kubernetes.Clientset, name if err != nil { return nil, err } - ips := make([]string, 0, len(pods.Items)*2) // no lint + ips := make([]string, 0, len(pods.Items)*2) //nolint for index := range pods.Items { for _, podIP := range pods.Items[index].Status.PodIPs { ips = append(ips, podIP.IP) diff --git a/test/validate/utils.go b/test/validate/utils.go index 3aff776ee2..4c81fe145a 100644 --- a/test/validate/utils.go +++ b/test/validate/utils.go @@ -47,7 +47,7 @@ func getPodIPsWithoutNodeIP(ctx context.Context, clientset *kubernetes.Clientset func contain(obj, target interface{}) bool { targetValue := reflect.ValueOf(target) - switch reflect.TypeOf(target).Kind() { // no lint + switch reflect.TypeOf(target).Kind() { //nolint case reflect.Slice, reflect.Array: for i := 0; i < targetValue.Len(); i++ { if targetValue.Index(i).Interface() == obj { From df934ecaa0ea541863dce4d74f386c9652de1f17 Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Sun, 25 Jun 2023 10:26:59 -0400 Subject: [PATCH 09/45] fix minor issues --- test/integration/manifests/goldpinger/daemonset-ipv6.yaml | 4 ++-- test/integration/manifests/load/noop-deployment.yaml | 3 ++- test/internal/k8sutils/utils.go | 1 + 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/test/integration/manifests/goldpinger/daemonset-ipv6.yaml b/test/integration/manifests/goldpinger/daemonset-ipv6.yaml index 77dfe0bf79..cf93e09ae3 100644 --- a/test/integration/manifests/goldpinger/daemonset-ipv6.yaml +++ b/test/integration/manifests/goldpinger/daemonset-ipv6.yaml @@ -59,8 +59,8 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP -# - name: HOSTS_TO_RESOLVE -# value: "1.1.1.1 8.8.8.8 www.bing.com" + - name: HOSTS_TO_RESOLVE + value: "2001:4860:4860::8888 www.bing.com" image: "docker.io/bloomberg/goldpinger:v3.7.0" securityContext: allowPrivilegeEscalation: false diff --git a/test/integration/manifests/load/noop-deployment.yaml b/test/integration/manifests/load/noop-deployment.yaml index 276d6f215f..e7990551a1 100644 --- a/test/integration/manifests/load/noop-deployment.yaml +++ b/test/integration/manifests/load/noop-deployment.yaml @@ -20,4 +20,5 @@ spec: securityContext: privileged: true nodeSelector: - kubernetes.io/os: linux \ No newline at end of file + kubernetes.io/os: linux + \ No newline at end of file diff --git a/test/internal/k8sutils/utils.go b/test/internal/k8sutils/utils.go index 66350062b9..5b208d153a 100644 --- a/test/internal/k8sutils/utils.go +++ b/test/internal/k8sutils/utils.go @@ -71,6 +71,7 @@ func mustParseResource(path string, out interface{}) error { if err := yaml.NewYAMLOrJSONDecoder(f, 0).Decode(out); err != nil { return err } + return err } From f5f96bb21195b5ce18bf7e6df5f28ce02253a99e Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Tue, 27 Jun 2023 14:02:24 -0400 Subject: [PATCH 10/45] change file name --- .../datapath/{datapath_win_test.go => datapath_windows_test.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/integration/datapath/{datapath_win_test.go => datapath_windows_test.go} (100%) diff --git a/test/integration/datapath/datapath_win_test.go b/test/integration/datapath/datapath_windows_test.go similarity index 100% rename from test/integration/datapath/datapath_win_test.go rename to test/integration/datapath/datapath_windows_test.go From 280493b4e260012378037061bdd79a4d0582386a Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Wed, 28 Jun 2023 15:39:28 -0400 Subject: [PATCH 11/45] fix conflicts --- .../cilium-overlay-load-test-template.yaml | 12 +- .../create-cluster-template.yaml | 5 +- .../pod-deployment-template.yaml | 9 +- .../restart-node-template.yaml | 6 +- .../validate-state-template.yaml | 6 +- .pipelines/cni/pipeline.yaml | 11 + .../windows-cni-load-test-template.yaml | 134 ++++++++++ .../aks-swift/e2e-step-template.yaml | 8 +- .../cilium-overlay-e2e-step-template.yaml | 8 +- .../cilium/cilium-e2e-step-template.yaml | 8 +- Makefile | 15 +- cni/build/windows.Dockerfile | 20 ++ hack/{swift => aks}/Makefile | 21 ++ hack/{swift => aks}/README.md | 1 + hack/{swift => aks}/kube-proxy.json | 0 hack/manifests/windows-update.yaml | 64 +++++ hack/scripts/updatecni.ps1 | 52 ++++ test/integration/load/load_test.go | 10 +- .../load/privileged-daemonset-windows.yaml | 32 +++ ...oyment.yaml => noop-deployment-linux.yaml} | 3 +- .../manifests/noop-deployment-windows.yaml | 23 ++ test/internal/k8sutils/utils.go | 4 +- test/validate/client.go | 2 + test/validate/windows_validate.go | 228 ++++++++++++++++++ 24 files changed, 646 insertions(+), 36 deletions(-) create mode 100644 .pipelines/cni/singletenancy/windows-cni-load-test-template.yaml create mode 100644 cni/build/windows.Dockerfile rename hack/{swift => aks}/Makefile (91%) rename hack/{swift => aks}/README.md (95%) rename hack/{swift => aks}/kube-proxy.json (100%) create mode 100644 hack/manifests/windows-update.yaml create mode 100644 hack/scripts/updatecni.ps1 create mode 100644 test/integration/manifests/load/privileged-daemonset-windows.yaml rename test/integration/manifests/{load/noop-deployment.yaml => noop-deployment-linux.yaml} (88%) create mode 100644 test/integration/manifests/noop-deployment-windows.yaml create mode 100644 test/validate/windows_validate.go diff --git a/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml b/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml index 9cf7a72a5c..044535bc6d 100644 --- a/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml +++ b/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml @@ -37,7 +37,7 @@ stages: inlineScript: | set -ex az extension add --name aks-preview - make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision) + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision) ls -lah pwd kubectl cluster-info @@ -70,6 +70,8 @@ stages: - template: ../load-test-templates/pod-deployment-template.yaml parameters: clusterName: ${{ parameters.clusterName }} + scaleup: 2400 + os: linux - stage: validate_state dependsOn: pod_deployment displayName: "Validate State" @@ -122,7 +124,7 @@ stages: addSpnToEnvironment: true inlineScript: | set -ex - make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision) + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision) name: "GetCluster" displayName: "Get AKS Cluster" - script: | @@ -149,9 +151,9 @@ stages: if [ "$(DELETE_RESOURCES)" ] then echo "Deleting Cluster and resource group" - make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision) - make -C ./hack/swift azcfg AZCLI=az REGION=$(LOCATION) - make -C ./hack/swift down AZCLI=az REGION=$(LOCATION) SUB=$(SUBSCRIPTION_ID) CLUSTER=${{ parameters.clusterName }}-$(make revision) + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision) + make -C ./hack/aks azcfg AZCLI=az REGION=$(LOCATION) + make -C ./hack/aks down AZCLI=az REGION=$(LOCATION) SUB=$(SUBSCRIPTION_ID) CLUSTER=${{ parameters.clusterName }}-$(make revision) echo "Cluster and resources down" else echo "Deletion of resources is False" diff --git a/.pipelines/cni/load-test-templates/create-cluster-template.yaml b/.pipelines/cni/load-test-templates/create-cluster-template.yaml index 251f20ce1a..de07665774 100644 --- a/.pipelines/cni/load-test-templates/create-cluster-template.yaml +++ b/.pipelines/cni/load-test-templates/create-cluster-template.yaml @@ -3,6 +3,7 @@ parameters: clusterName: "" nodeCount: "" vmSize: "" + windowsVMSize: "" steps: - task: AzureCLI@1 @@ -13,7 +14,7 @@ steps: addSpnToEnvironment: true inlineScript: | set -ex - make -C ./hack/swift azcfg AZCLI=az REGION=$(LOCATION) - make -C ./hack/swift ${{ parameters.clusterType }} AZCLI=az REGION=$(LOCATION) SUB=$(SUBSCRIPTION_ID) CLUSTER=${{ parameters.clusterName }}-$(make revision) NODE_COUNT=${{ parameters.nodeCount }} VM_SIZE=${{ parameters.vmSize }} + make -C ./hack/aks azcfg AZCLI=az REGION=$(LOCATION) + make -C ./hack/aks ${{ parameters.clusterType }} AZCLI=az REGION=$(LOCATION) SUB=$(SUBSCRIPTION_ID) CLUSTER=${{ parameters.clusterName }}-$(make revision) NODE_COUNT=${{ parameters.nodeCount }} VM_SIZE=${{ parameters.vmSize }} WINDOWS_VM_SKU=${{ parameters.windowsVMSize }} WINDOWS_USERNAME=${WINDOWS_USERNAME} WINDOWS_PASSWORD=${WINDOWS_PASSWORD} name: "CreateAksCluster" displayName: "Create AKS Cluster" diff --git a/.pipelines/cni/load-test-templates/pod-deployment-template.yaml b/.pipelines/cni/load-test-templates/pod-deployment-template.yaml index fd3dfca680..6032c028d2 100644 --- a/.pipelines/cni/load-test-templates/pod-deployment-template.yaml +++ b/.pipelines/cni/load-test-templates/pod-deployment-template.yaml @@ -1,6 +1,9 @@ parameters: clusterName: "" - + scaleup: 1000 + os: "" + iterations: 4 + steps: - task: AzureCLI@1 displayName: "Pod Deployment" @@ -12,6 +15,6 @@ steps: inlineScript: | set -ex az extension add --name aks-preview - make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision) + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision) cd test/integration/load - go test -timeout 30m -tags load -run ^TestLoad$ -tags=load -iterations=4 -scaleup=2400 + go test -timeout 30m -tags load -run ^TestLoad$ -tags=load -iterations=${{ parameters.iterations }} -scaleup=${{ parameters.scaleup }} -os=${{ parameters.os }} diff --git a/.pipelines/cni/load-test-templates/restart-node-template.yaml b/.pipelines/cni/load-test-templates/restart-node-template.yaml index caf63196d2..4009150b8a 100644 --- a/.pipelines/cni/load-test-templates/restart-node-template.yaml +++ b/.pipelines/cni/load-test-templates/restart-node-template.yaml @@ -11,15 +11,15 @@ steps: inlineScript: | echo "Scale up the pods and immediated restart the nodes" clusterName=${{ parameters.clusterName }}-$(make revision) - make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${clusterName} - make -C ./hack/swift azcfg AZCLI=az REGION=$(LOCATION) + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${clusterName} + make -C ./hack/aks azcfg AZCLI=az REGION=$(LOCATION) cd test/integration/load echo "Scaling the pods down to 100 per node" go test -count 1 -timeout 30m -tags load -run ^TestScaleDeployment$ -tags=load -scaleup=1000 -skip-wait=true cd ../../../ echo "Restarting the nodes" vmss_name=$(az vmss list -g MC_${clusterName}_${clusterName}_$(LOCATION) --query "[].name" -o tsv) - make -C ./hack/swift restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(LOCATION) VMSS_NAME=$vmss_name + make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(LOCATION) VMSS_NAME=$vmss_name cd test/integration/load go test -count 1 -timeout 30m -tags load -run ^TestScaleDeployment$ -tags=load -replicas=1000 name: "RestartNodes" diff --git a/.pipelines/cni/load-test-templates/validate-state-template.yaml b/.pipelines/cni/load-test-templates/validate-state-template.yaml index 8b4ab0102d..8f0ae209a2 100644 --- a/.pipelines/cni/load-test-templates/validate-state-template.yaml +++ b/.pipelines/cni/load-test-templates/validate-state-template.yaml @@ -1,5 +1,6 @@ parameters: clusterName: "" + os: "linux" restartCase: "false" steps: @@ -10,10 +11,9 @@ steps: scriptType: "bash" addSpnToEnvironment: true inlineScript: | - export RESTART_CASE=${{ parameters.restartCase }} - make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision) + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision) kubectl get pods -A - make test-validate-state + make test-validate-state OS=${{ parameters.os }} RESTART_CASE=${{ parameters.restartCase }} name: "ValidateState" displayName: "Validate State" retryCountOnTaskFailure: 3 diff --git a/.pipelines/cni/pipeline.yaml b/.pipelines/cni/pipeline.yaml index afad7accc1..3fda84f807 100644 --- a/.pipelines/cni/pipeline.yaml +++ b/.pipelines/cni/pipeline.yaml @@ -24,3 +24,14 @@ stages: nodeCount: 10 vmSize: "Standard_DS4_v2" dependsOn: setup + - template: singletenancy/windows-cni-load-test-template.yaml + parameters: + name: win_cniv1 + clusterType: "windows-cniv1-up" + clusterName: "win-cniv1" + nodeCount: 2 + vmSize: "Standard_B2s" + dependsOn: setup + windowsVMSize: ${WINDOWS_VM_SKU} + os: windows + cni: cniv1 diff --git a/.pipelines/cni/singletenancy/windows-cni-load-test-template.yaml b/.pipelines/cni/singletenancy/windows-cni-load-test-template.yaml new file mode 100644 index 0000000000..6c46d5acb3 --- /dev/null +++ b/.pipelines/cni/singletenancy/windows-cni-load-test-template.yaml @@ -0,0 +1,134 @@ +parameters: + dependsOn: "" + name: "" + clusterType: "" + clusterName: "" + nodeCount: "" + vmSize: "" + windowsVMSize: "" + os: "" + cni: "" + +stages: + - stage: createAKSclusterWindows + dependsOn: ${{ parameters.dependsOn }} + displayName: "Windows AKS Cluster ${{ parameters.cni }}" + jobs: + - job: create_aks_cluster_with_${{ parameters.name }} + steps: + - template: ../load-test-templates/create-cluster-template.yaml + parameters: + clusterType: ${{ parameters.clusterType }} + clusterName: ${{ parameters.clusterName }} + nodeCount: ${{ parameters.nodeCount }} + vmSize: ${{ parameters.vmSize }} + windowsVMSize: ${{ parameters.windowsVMSize }} + - stage: build_images + dependsOn: ${{ parameters.dependsOn }} + displayName: "Build CNI Images" + jobs: + - job: build_cni_images + pool: + name: "$(BUILD_POOL_NAME_LINUX_AMD64)" + strategy: + matrix: + windows_cniv1_amd64: + os: windows + name: cni-plugin + arch: amd64 + os_version: ltsc2022 + steps: + - template: ../../containers/container-template.yaml + parameters: + arch: $(arch) + name: $(name) + os: $(os) + os_version: $(os_version) + - stage: update_cni + dependsOn: + - createAKSclusterWindows + - build_images + displayName: "Update CNI on Cluster" + jobs: + - job: deploy_pods + strategy: + matrix: + windows_cniv1_amd64: + os: windows + arch: amd64 + os_version: ltsc2022 + steps: + - task: AzureCLI@1 + inputs: + azureSubscription: $(TEST_SUB_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + set -ex + export CNI_IMAGE=$(make acncli-image-name-and-tag OS=$(os) ARCH=$(arch) OS_VERSION=$(os_version)) + az extension add --name aks-preview + clusterName=${{ parameters.clusterName }}-$(make revision) + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${clusterName} + make -C ./hack/aks azcfg AZCLI=az REGION=$(LOCATION) + envsubst < ./hack/manifests/windows-update.yaml | kubectl apply -f - + name: "UploadCNI" + displayName: "Upload CNI" + - script: | + set -ex + kubectl rollout status daemonset/azure-cni-windows -n kube-system + kubectl get pods -A + name: "WaitForCNI" + displayName: "Wait For CNI" + - stage: pod_deployment_windows + dependsOn: update_cni + displayName: "Pod Deployment" + jobs: + - job: deploy_pods + steps: + - template: ../load-test-templates/pod-deployment-template.yaml + parameters: + clusterName: ${{ parameters.clusterName }} + scaleup: ${WINDOWS_SCALEUP} + os: ${{ parameters.os }} + cni: ${{ parameters.cni }} + iterations: ${WINDOWS_ITERATIONS} + - stage: validate_state_windows + dependsOn: pod_deployment_windows + displayName: "Validate State" + jobs: + - job: validate_state + steps: + - template: ../load-test-templates/validate-state-template.yaml + parameters: + clusterName: ${{ parameters.clusterName }} + os: ${{ parameters.os }} + cni: ${{ parameters.cni }} + - stage: delete_resources + displayName: "Delete Resources" + dependsOn: + - validate_state_windows + jobs: + - job: delete_resources + steps: + - task: AzureCLI@1 + inputs: + azureSubscription: $(TEST_SUB_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + set -ex + if [ "$(DELETE_RESOURCES)" ] + then + echo "Deleting Cluster and resource group" + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision) + make -C ./hack/aks azcfg AZCLI=az REGION=$(LOCATION) + make -C ./hack/aks down AZCLI=az REGION=$(LOCATION) SUB=$(SUBSCRIPTION_ID) CLUSTER=${{ parameters.clusterName }}-$(make revision) + echo "Cluster and resources down" + else + echo "Deletion of resources is False" + fi + name: "CleanUpCluster" + displayName: "Cleanup cluster" + condition: always() diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml index 1753774243..c5ad773e4b 100644 --- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml +++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml @@ -32,8 +32,8 @@ steps: inlineScript: | mkdir -p ~/.kube/ echo "Create AKS cluster" - make -C ./hack/swift azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) - make -C ./hack/swift byocni-up AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-${{ parameters.osSku }}-$(make revision) OSSKU=${{ parameters.osSku }} + make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) + make -C ./hack/aks byocni-up AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-${{ parameters.osSku }}-$(make revision) OSSKU=${{ parameters.osSku }} echo "Cluster successfully created" displayName: Create test cluster condition: succeeded() @@ -124,8 +124,8 @@ steps: addSpnToEnvironment: true inlineScript: | echo "Deleting cluster" - make -C ./hack/swift azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) - make -C ./hack/swift down AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-${{ parameters.osSku }}-$(make revision) + make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) + make -C ./hack/aks down AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-${{ parameters.osSku }}-$(make revision) echo "Cluster and resources down" name: "Cleanupcluster" displayName: "Cleanup cluster" diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml index 9ed3b09a82..0fbf47af48 100644 --- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml @@ -31,8 +31,8 @@ steps: inlineScript: | mkdir -p ~/.kube/ echo "Create AKS Overlay cluster" - make -C ./hack/swift azcfg AZCLI=az REGION=$(REGION_OVERLAY_CLUSTER_TEST) - make -C ./hack/swift overlay-no-kube-proxy-up AZCLI=az REGION=$(REGION_OVERLAY_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) VM_SIZE=Standard_B2ms + make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_OVERLAY_CLUSTER_TEST) + make -C ./hack/aks overlay-no-kube-proxy-up AZCLI=az REGION=$(REGION_OVERLAY_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) VM_SIZE=Standard_B2ms echo "Cluster successfully created" displayName: Create Overlay cluster condition: succeeded() @@ -189,8 +189,8 @@ steps: addSpnToEnvironment: true inlineScript: | echo "Deleting cluster" - make -C ./hack/swift azcfg AZCLI=az - make -C ./hack/swift down SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision) + make -C ./hack/aks azcfg AZCLI=az + make -C ./hack/aks down SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision) echo "Cluster and resources down" name: "Cleanupcluster" displayName: "Cleanup cluster" diff --git a/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml b/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml index 3fefd72ecd..7f4404d879 100644 --- a/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml +++ b/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml @@ -31,8 +31,8 @@ steps: inlineScript: | mkdir -p ~/.kube/ echo "Create AKS cluster" - make -C ./hack/swift azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) - make -C ./hack/swift swift-no-kube-proxy-up AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) VM_SIZE=Standard_B2ms + make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) + make -C ./hack/aks swift-no-kube-proxy-up AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) VM_SIZE=Standard_B2ms echo "Cluster successfully created" displayName: Create test cluster condition: succeeded() @@ -173,8 +173,8 @@ steps: addSpnToEnvironment: true inlineScript: | echo "Deleting cluster" - make -C ./hack/swift azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) - make -C ./hack/swift down AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) + make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) + make -C ./hack/aks down AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) echo "Cluster and resources down" name: "Cleanupcluster" displayName: "Cleanup cluster" diff --git a/Makefile b/Makefile index 3933d0c206..6ed78f0e8b 100644 --- a/Makefile +++ b/Makefile @@ -438,6 +438,19 @@ npm-image-pull: ## pull cns container image. IMAGE=$(NPM_IMAGE) \ TAG=$(NPM_PLATFORM_TAG) +# cni-plugin + +cni-plugin-image: ## build cni plugin container image. + $(MAKE) container \ + DOCKERFILE=cni/build/$(OS).Dockerfile \ + IMAGE=$(ACNCLI_IMAGE) \ + EXTRA_BUILD_ARGS='--build-arg CNI_AI_PATH=$(CNI_AI_PATH) --build-arg CNI_AI_ID=$(CNI_AI_ID) --build-arg OS_VERSION=$(OS_VERSION)' \ + PLATFORM=$(PLATFORM) \ + TAG=$(ACNCLI_PLATFORM_TAG) \ + OS=$(OS) \ + ARCH=$(ARCH) \ + OS_VERSION=$(OS_VERSION) + ## Legacy @@ -724,7 +737,7 @@ test-integration: ## run all integration tests. go test -mod=readonly -buildvcs=false -timeout 1h -coverpkg=./... -race -covermode atomic -coverprofile=coverage.out -tags=integration ./test/integration... test-validate-state: - cd test/integration/load && go test -count 1 -timeout 30m -tags load -run ^TestValidateState -tags=load -restart-case=$(RESTART_CASE) + cd test/integration/load && go test -count 1 -timeout 30m -tags load -run ^TestValidateState -tags=load -restart-case=$(RESTART_CASE) -os=$(OS) cd ../../.. test-cyclonus: ## run the cyclonus test for npm. diff --git a/cni/build/windows.Dockerfile b/cni/build/windows.Dockerfile new file mode 100644 index 0000000000..036081a6e5 --- /dev/null +++ b/cni/build/windows.Dockerfile @@ -0,0 +1,20 @@ +ARG OS_VERSION +FROM --platform=linux/amd64 mcr.microsoft.com/oss/go/microsoft/golang:1.20 AS builder +ARG VERSION +ARG CNI_AI_PATH +ARG CNI_AI_ID +WORKDIR /azure-container-networking +COPY . . +RUN GOOS=windows CGO_ENABLED=0 go build -a -o azure-vnet.exe -trimpath -ldflags "-X main.version="$VERSION"" -gcflags="-dwarflocationlists=true" cni/network/plugin/main.go +RUN GOOS=windows CGO_ENABLED=0 go build -a -o azure-vnet-telemetry.exe -trimpath -ldflags "-X main.version="$VERSION" -X "$CNI_AI_PATH"="$CNI_AI_ID"" -gcflags="-dwarflocationlists=true" cni/telemetry/service/telemetrymain.go +RUN GOOS=windows CGO_ENABLED=0 go build -a -o azure-vnet-ipam.exe -trimpath -ldflags "-X main.version="$VERSION"" -gcflags="-dwarflocationlists=true" cni/ipam/plugin/main.go + +FROM mcr.microsoft.com/windows/servercore:${OS_VERSION} +SHELL ["powershell", "-command"] +COPY --from=builder /azure-container-networking/azure-vnet.exe azure-vnet.exe +COPY --from=builder /azure-container-networking/azure-vnet-telemetry.exe azure-vnet-telemetry.exe +COPY --from=builder /azure-container-networking/azure-vnet-ipam.exe azure-vnet-ipam.exe + +# This would be replaced with dropgz version of windows. +COPY --from=builder /azure-container-networking/hack/scripts/updatecni.ps1 updatecni.ps1 +ENTRYPOINT ["powershell.exe", ".\\updatecni.ps1"] diff --git a/hack/swift/Makefile b/hack/aks/Makefile similarity index 91% rename from hack/swift/Makefile rename to hack/aks/Makefile index 73d8074e1b..b16019b1dc 100644 --- a/hack/swift/Makefile +++ b/hack/aks/Makefile @@ -190,6 +190,27 @@ swift-up: rg-up swift-net-up ## Bring up a SWIFT AzCNI cluster --yes @$(MAKE) set-kubeconf +windows-cniv1-up: rg-up overlay-net-up ## Bring up a Windows CNIv1 cluster + $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ + --node-count $(NODE_COUNT) \ + --node-vm-size $(VM_SIZE) \ + --network-plugin azure \ + --windows-admin-password $(WINDOWS_PASSWORD) \ + --windows-admin-username $(WINDOWS_USERNAME) \ + --vnet-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/nodenet \ + --no-ssh-key \ + --yes + + $(AZCLI) aks nodepool add --resource-group $(GROUP) --cluster-name $(CLUSTER) \ + --os-type Windows \ + --os-sku Windows2022 \ + --max-pods 250 \ + --name npwin \ + --node-count $(NODE_COUNT) \ + -s $(WINDOWS_VM_SKU) + + @$(MAKE) set-kubeconf + down: ## Delete the cluster $(AZCLI) aks delete -g $(GROUP) -n $(CLUSTER) --yes @$(MAKE) unset-kubeconf diff --git a/hack/swift/README.md b/hack/aks/README.md similarity index 95% rename from hack/swift/README.md rename to hack/aks/README.md index 40069d88a3..3a4e80e4f0 100644 --- a/hack/swift/README.md +++ b/hack/aks/README.md @@ -28,6 +28,7 @@ AKS Clusters swift-byocni-up Bring up a SWIFT BYO CNI cluster swift-cilium-up Bring up a SWIFT Cilium cluster swift-up Bring up a SWIFT AzCNI cluster + windows-cniv1-up Bring up a Windows AzCNIv1 cluster down Delete the cluster vmss-restart Restart the nodes of the cluster ``` diff --git a/hack/swift/kube-proxy.json b/hack/aks/kube-proxy.json similarity index 100% rename from hack/swift/kube-proxy.json rename to hack/aks/kube-proxy.json diff --git a/hack/manifests/windows-update.yaml b/hack/manifests/windows-update.yaml new file mode 100644 index 0000000000..2f21f31516 --- /dev/null +++ b/hack/manifests/windows-update.yaml @@ -0,0 +1,64 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: azure-cni-windows + labels: + tier: node + app: azure-cni + namespace: kube-system +spec: + selector: + matchLabels: + app: azure-cni + template: + metadata: + labels: + tier: node + app: azure-cni + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: In + values: + - windows + - key: kubernetes.io/arch + operator: In + values: + - amd64 + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\system" + hostNetwork: true + serviceAccountName: azure-cni + tolerations: + - operator: Exists + effect: NoSchedule + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + containers: + - name: cni-drop + image: ${CNI_IMAGE} + imagePullPolicy: Always + volumeMounts: + - name: cni-bin + mountPath: /k/azurecni/bin/ + volumes: + - name: cni-bin + hostPath: + path: /k/azurecni/bin + type: DirectoryOrCreate +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: azure-cni + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: EnsureExists diff --git a/hack/scripts/updatecni.ps1 b/hack/scripts/updatecni.ps1 new file mode 100644 index 0000000000..4ee26376ab --- /dev/null +++ b/hack/scripts/updatecni.ps1 @@ -0,0 +1,52 @@ +Write-Host $env:CONTAINER_SANDBOX_MOUNT_POINT +$sourceCNI = $env:CONTAINER_SANDBOX_MOUNT_POINT + "azure-vnet.exe" +$sourceIpam = $env:CONTAINER_SANDBOX_MOUNT_POINT + "azure-vnet-ipam.exe" +$sourceTelemetry = $env:CONTAINER_SANDBOX_MOUNT_POINT + "azure-vnet-telemetry.exe" + +$sourceCNIVersion = & "$sourceCNI" -v +$currentVersion = "" +$sourceTelemetryVersion = & "$sourceTelemetry" -v +$currentTelemetryVersion = "" + +$cniExists = Test-Path "C:\k\azurecni\bin\azure-vnet.exe" +$telemetryExists = Test-Path "C:\k\azurecni\bin\azure-vnet-telemetry.exe" + +Write-Host "Source $sourceCNIVersion" +Write-Host "Source Telemetry $sourceTelemetryVersion" + +if ($cniExists) { + $currentVersion = & "C:\k\azurecni\bin\azure-vnet.exe" -v +} + +if($telemetryExists){ + $currentTelemetryVersion = & "C:\k\azurecni\bin\azure-vnet-telemetry.exe" -v +} + + +Write-Host "Current Host $currentVersion" +Write-Host "Current Telemetry $currentTelemetryVersion" + +## check telemetry was already installed so not to get stuck in a infinite loop of rebooting and killing the process +if ($currentTelemetryVersion -ne $sourceTelemetryVersion){ + $processes = Get-Process -Name azure-vnet-telemetry -ErrorAction SilentlyContinue + for ($i = 0; $i -lt $processes.Count; $i++) { + Write-Host "Killing azure-vnet-telemetry process..." + $processes[$i].Kill() + } + Write-Host "copying azure-vnet-telemetry to windows node..." + Remove-Item "C:\k\azurecni\bin\azure-vnet-telemetry.exe" + Copy-Item $sourceTelemetry -Destination "C:\k\azurecni\bin" +} + +## check CNI was already installed so not to get stuck in a infinite loop of rebooting +if ($currentVersion -ne $sourceCNIVersion){ + Write-Host "copying azure-vnet to windows node..." + Remove-Item "C:\k\azurecni\bin\azure-vnet.exe" + Copy-Item $sourceCNI -Destination "C:\k\azurecni\bin" + + Write-Host "copying azure-vnet-ipam to windows node..." + Remove-Item "C:\k\azurecni\bin\azure-vnet-ipam.exe" + Copy-Item $sourceIpam -Destination "C:\k\azurecni\bin" +} + +Start-Sleep -s 1000 diff --git a/test/integration/load/load_test.go b/test/integration/load/load_test.go index bc49ca23a5..c734897256 100644 --- a/test/integration/load/load_test.go +++ b/test/integration/load/load_test.go @@ -14,7 +14,6 @@ import ( const ( manifestDir = "../manifests" - noopdeployment = manifestDir + "/load/noop-deployment.yaml" podLabelSelector = "load-test=true" ) @@ -32,6 +31,11 @@ var ( validateDualStack = flag.Bool("validate-dualstack", false, "Validate the dualstack overlay") ) +var noopDeploymentMap = map[string]string{ + "windows": manifestDir + "/noop-deployment-windows.yaml", + "linux": manifestDir + "/noop-deployment-linux.yaml", +} + /* In order to run the scale tests, you need a k8s cluster and its kubeconfig. If no kubeconfig is passed, the test will attempt to find one in the default location for kubectl config. @@ -76,7 +80,7 @@ func TestLoad(t *testing.T) { } } - deployment, err := k8sutils.MustParseDeployment(noopdeployment) + deployment, err := k8sutils.MustParseDeployment(noopDeploymentMap[*osType]) if err != nil { t.Fatal(err) } @@ -171,7 +175,7 @@ func TestScaleDeployment(t *testing.T) { } } - deployment, err := k8sutils.MustParseDeployment(noopdeployment) + deployment, err := k8sutils.MustParseDeployment(noopDeploymentMap[*osType]) if err != nil { t.Fatal(err) } diff --git a/test/integration/manifests/load/privileged-daemonset-windows.yaml b/test/integration/manifests/load/privileged-daemonset-windows.yaml new file mode 100644 index 0000000000..c9a7839013 --- /dev/null +++ b/test/integration/manifests/load/privileged-daemonset-windows.yaml @@ -0,0 +1,32 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: privileged-daemonset + namespace: kube-system + labels: + app: privileged-daemonset +spec: + selector: + matchLabels: + app: privileged-daemonset + template: + metadata: + labels: + app: privileged-daemonset + spec: + nodeSelector: + kubernetes.io/os: windows + containers: + - name: powershell + image: mcr.microsoft.com/powershell:lts-nanoserver-1809 + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\SYSTEM" + command: + - powershell.exe + - -command + - | + while ($true) { Start-Sleep -Seconds 2147483 } + hostNetwork: true + terminationGracePeriodSeconds: 0 diff --git a/test/integration/manifests/load/noop-deployment.yaml b/test/integration/manifests/noop-deployment-linux.yaml similarity index 88% rename from test/integration/manifests/load/noop-deployment.yaml rename to test/integration/manifests/noop-deployment-linux.yaml index e7990551a1..6b12793189 100644 --- a/test/integration/manifests/load/noop-deployment.yaml +++ b/test/integration/manifests/noop-deployment-linux.yaml @@ -20,5 +20,4 @@ spec: securityContext: privileged: true nodeSelector: - kubernetes.io/os: linux - \ No newline at end of file + "kubernetes.io/os": linux diff --git a/test/integration/manifests/noop-deployment-windows.yaml b/test/integration/manifests/noop-deployment-windows.yaml new file mode 100644 index 0000000000..96619555c9 --- /dev/null +++ b/test/integration/manifests/noop-deployment-windows.yaml @@ -0,0 +1,23 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: win-load-test + namespace: load-test +spec: + selector: + matchLabels: + os: windows + load-test: "true" + template: + metadata: + labels: + os: windows + load-test: "true" + spec: + containers: + - name: noop + image: mcr.microsoft.com/windows/nanoserver:ltsc2022 + ports: + - containerPort: 80 + nodeSelector: + "kubernetes.io/os": windows diff --git a/test/internal/k8sutils/utils.go b/test/internal/k8sutils/utils.go index 5b208d153a..95041ca693 100644 --- a/test/internal/k8sutils/utils.go +++ b/test/internal/k8sutils/utils.go @@ -340,7 +340,7 @@ func ExecCmdOnPod(ctx context.Context, clientset *kubernetes.Clientset, namespac Stdin: false, Stdout: true, Stderr: true, - TTY: true, + TTY: false, }, scheme.ParameterCodec) exec, err := remotecommand.NewSPDYExecutor(config, "POST", req.URL()) @@ -353,7 +353,7 @@ func ExecCmdOnPod(ctx context.Context, clientset *kubernetes.Clientset, namespac Stdin: nil, Stdout: &stdout, Stderr: &stderr, - Tty: true, + Tty: false, }) if err != nil { return []byte{}, errors.Wrapf(err, "error in executing command %s", cmd) diff --git a/test/validate/client.go b/test/validate/client.go index 11b52081e5..0ede9bbb5c 100644 --- a/test/validate/client.go +++ b/test/validate/client.go @@ -34,6 +34,8 @@ func GetValidatorClient(os string) validatorClient { switch os { case "linux": return &LinuxClient{} + case "windows": + return &WindowsClient{} default: return nil } diff --git a/test/validate/windows_validate.go b/test/validate/windows_validate.go new file mode 100644 index 0000000000..9e54f61bef --- /dev/null +++ b/test/validate/windows_validate.go @@ -0,0 +1,228 @@ +package validate + +import ( + "context" + "encoding/json" + "log" + "net" + + k8sutils "github.com/Azure/azure-container-networking/test/internal/k8sutils" + "github.com/pkg/errors" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" +) + +const ( + privilegedWindowsDaemonSetPath = "../manifests/load/privileged-daemonset-windows.yaml" + windowsNodeSelector = "kubernetes.io/os=windows" +) + +var ( + hnsEndPpointCmd = []string{"powershell", "-c", "Get-HnsEndpoint | ConvertTo-Json"} + azureVnetCmd = []string{"powershell", "-c", "cat ../../k/azure-vnet.json"} + azureVnetIpamCmd = []string{"powershell", "-c", "cat ../../k/azure-vnet-ipam.json"} +) + +type WindowsClient struct{} + +type WindowsValidator struct { + Validator +} + +type HNSEndpoint struct { + MacAddress string `json:"MacAddress"` + IPAddress net.IP `json:"IPAddress"` + IPv6Address net.IP `json:",omitempty"` + IsRemoteEndpoint bool `json:",omitempty"` +} + +type AzureVnet struct { + NetworkInfo NetworkInfo `json:"Network"` +} + +type NetworkInfo struct { + ExternalInterfaces map[string]ExternalInterface `json:"ExternalInterfaces"` +} + +type ExternalInterface struct { + Networks map[string]Network `json:"Networks"` +} + +type Network struct { + Endpoints map[string]Endpoint `json:"Endpoints"` +} + +type Endpoint struct { + IPAddresses []net.IPNet `json:"IPAddresses"` + IfName string `json:"IfName"` +} + +type AzureVnetIpam struct { + IPAM AddressSpaces `json:"IPAM"` +} + +type AddressSpaces struct { + AddrSpaces map[string]AddressSpace `json:"AddressSpaces"` +} + +type AddressSpace struct { + Pools map[string]AddressPool `json:"Pools"` +} + +type AddressPool struct { + Addresses map[string]AddressRecord `json:"Addresses"` +} + +type AddressRecord struct { + Addr net.IP + InUse bool +} + +func (w *WindowsClient) CreateClient(ctx context.Context, clienset *kubernetes.Clientset, config *rest.Config, namespace, cni string, restartCase bool) IValidator { + // deploy privileged pod + privilegedDaemonSet, err := k8sutils.MustParseDaemonSet(privilegedWindowsDaemonSetPath) + if err != nil { + panic(err) + } + daemonsetClient := clienset.AppsV1().DaemonSets(privilegedNamespace) + err = k8sutils.MustCreateDaemonset(ctx, daemonsetClient, privilegedDaemonSet) + if err != nil { + panic(err) + } + err = k8sutils.WaitForPodsRunning(ctx, clienset, privilegedNamespace, privilegedLabelSelector) + if err != nil { + panic(err) + } + return &WindowsValidator{ + Validator: Validator{ + ctx: ctx, + clientset: clienset, + config: config, + namespace: namespace, + cni: cni, + restartCase: restartCase, + }, + } +} + +func (v *WindowsValidator) ValidateStateFile() error { + checks := []struct { + name string + stateFileIps func([]byte) (map[string]string, error) + podLabelSelector string + podNamespace string + cmd []string + }{ + {"hns", hnsStateFileIps, privilegedLabelSelector, privilegedNamespace, hnsEndPpointCmd}, + {"azure-vnet", azureVnetIps, privilegedLabelSelector, privilegedNamespace, azureVnetCmd}, + {"azure-vnet-ipam", azureVnetIpamIps, privilegedLabelSelector, privilegedNamespace, azureVnetIpamCmd}, + } + + for _, check := range checks { + err := v.validate(check.stateFileIps, check.cmd, check.name, check.podNamespace, check.podLabelSelector) + if err != nil { + return err + } + } + return nil +} + +func hnsStateFileIps(result []byte) (map[string]string, error) { + var hnsResult []HNSEndpoint + err := json.Unmarshal(result, &hnsResult) + if err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal hns endpoint list") + } + + hnsPodIps := make(map[string]string) + for _, v := range hnsResult { + if !v.IsRemoteEndpoint { + hnsPodIps[v.IPAddress.String()] = v.MacAddress + } + } + return hnsPodIps, nil +} + +func azureVnetIps(result []byte) (map[string]string, error) { + var azureVnetResult AzureVnet + err := json.Unmarshal(result, &azureVnetResult) + if err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal azure vnet") + } + + azureVnetPodIps := make(map[string]string) + for _, v := range azureVnetResult.NetworkInfo.ExternalInterfaces { + for _, v := range v.Networks { + for _, e := range v.Endpoints { + for _, v := range e.IPAddresses { + azureVnetPodIps[v.IP.String()] = e.IfName + } + } + } + } + return azureVnetPodIps, nil +} + +func azureVnetIpamIps(result []byte) (map[string]string, error) { + var azureVnetIpamResult AzureVnetIpam + err := json.Unmarshal(result, &azureVnetIpamResult) + if err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal azure vnet ipam") + } + + azureVnetIpamPodIps := make(map[string]string) + + for _, v := range azureVnetIpamResult.IPAM.AddrSpaces { + for _, v := range v.Pools { + for _, v := range v.Addresses { + if v.InUse { + azureVnetIpamPodIps[v.Addr.String()] = v.Addr.String() + } + } + } + } + return azureVnetIpamPodIps, nil +} + +func (v *WindowsValidator) validate(stateFileIps stateFileIpsFunc, cmd []string, checkType, namespace, labelSelector string) error { + log.Println("Validating ", checkType, " state file") + nodes, err := k8sutils.GetNodeListByLabelSelector(v.ctx, v.clientset, windowsNodeSelector) + if err != nil { + return errors.Wrapf(err, "failed to get node list") + } + for index := range nodes.Items { + // get the privileged pod + pod, err := k8sutils.GetPodsByNode(v.ctx, v.clientset, namespace, labelSelector, nodes.Items[index].Name) + if err != nil { + return errors.Wrapf(err, "failed to get privileged pod") + } + podName := pod.Items[0].Name + // exec into the pod to get the state file + result, err := k8sutils.ExecCmdOnPod(v.ctx, v.clientset, namespace, podName, cmd, v.config) + if err != nil { + return errors.Wrapf(err, "failed to exec into privileged pod") + } + filePodIps, err := stateFileIps(result) + if err != nil { + return errors.Wrapf(err, "failed to get pod ips from state file") + } + if len(filePodIps) == 0 && v.restartCase { + log.Printf("No pods found on node %s", nodes.Items[index].Name) + continue + } + // get the pod ips + podIps := getPodIPsWithoutNodeIP(v.ctx, v.clientset, nodes.Items[index]) + + check := compareIPs(filePodIps, podIps) + + if !check { + return errors.Wrapf(errors.New("State file validation failed"), "for %s on node %s", checkType, nodes.Items[index].Name) + } + } + log.Printf("State file validation for %s passed", checkType) + return nil +} + +func (v *WindowsValidator) ValidateRestartNetwork() error { + return nil +} From 8d3ce618ffb95fbbfd888b5c1c7320ada21cc06d Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Wed, 28 Jun 2023 15:40:25 -0400 Subject: [PATCH 12/45] fix conflicts --- test/integration/manifests/load/privileged-daemonset.yaml | 4 ++++ test/integration/manifests/noop-deployment-linux.yaml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/test/integration/manifests/load/privileged-daemonset.yaml b/test/integration/manifests/load/privileged-daemonset.yaml index 519c81d3f2..77244f030a 100644 --- a/test/integration/manifests/load/privileged-daemonset.yaml +++ b/test/integration/manifests/load/privileged-daemonset.yaml @@ -33,7 +33,11 @@ spec: volumes: - name: azure-cns hostPath: +<<<<<<< HEAD path: /var/run/azure-network +======= + path: /var/run/azure-cns +>>>>>>> d691eef5 (fix conflicts) - name: azure-cns-noncilium hostPath: path: /var/run diff --git a/test/integration/manifests/noop-deployment-linux.yaml b/test/integration/manifests/noop-deployment-linux.yaml index 6b12793189..4d4acd89c2 100644 --- a/test/integration/manifests/noop-deployment-linux.yaml +++ b/test/integration/manifests/noop-deployment-linux.yaml @@ -20,4 +20,4 @@ spec: securityContext: privileged: true nodeSelector: - "kubernetes.io/os": linux + kubernetes.io/os: linux From f76a6ab415e6129f111694e2d81af8bd92a73adb Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Wed, 28 Jun 2023 15:38:00 -0400 Subject: [PATCH 13/45] fix conflicts --- test/integration/manifests/noop-deployment-linux.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/test/integration/manifests/noop-deployment-linux.yaml b/test/integration/manifests/noop-deployment-linux.yaml index 4d4acd89c2..73ea8b26cf 100644 --- a/test/integration/manifests/noop-deployment-linux.yaml +++ b/test/integration/manifests/noop-deployment-linux.yaml @@ -21,3 +21,4 @@ spec: privileged: true nodeSelector: kubernetes.io/os: linux + From da5879a7adb71612d17f70ba99d3152fb3ea7868 Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Thu, 29 Jun 2023 12:12:04 -0400 Subject: [PATCH 14/45] integrate #1980 to one PR --- .pipelines/pipeline.yaml | 8 ++ .../dualstackoverlay-e2e-job-template.yaml | 32 ++++++++ .../dualstackoverlay-e2e-step-template.yaml | 76 +++++++++++++++++++ hack/aks/Makefile | 25 ++++++ hack/aks/README.md | 22 +++--- 5 files changed, 153 insertions(+), 10 deletions(-) create mode 100644 .pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-job-template.yaml create mode 100644 .pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml index e55af8c9e8..608703ef76 100644 --- a/.pipelines/pipeline.yaml +++ b/.pipelines/pipeline.yaml @@ -321,6 +321,14 @@ stages: testDropgz: "" clusterName: "overlaye2e" + - template: singletenancy/dualstack-overlay/dualstackoverlay-e2e-job-template.yaml + parameters: + name: "dualstackoverlay_e2e" + displayName: AKS DualStack Overlay + pipelineBuildImage: "$(BUILD_IMAGE)" + testDropgz: "" + clusterName: "dualstackoverlaye2e" + - template: singletenancy/aks-swift/e2e-job-template.yaml parameters: name: "aks_swift_e2e" diff --git a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-job-template.yaml b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-job-template.yaml new file mode 100644 index 0000000000..fadb3196ab --- /dev/null +++ b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-job-template.yaml @@ -0,0 +1,32 @@ +parameters: + name: "" + displayName: "" + pipelineBuildImage: "$(BUILD_IMAGE)" + testDropgz: "" + clusterName: "" + +stages: + - stage: ${{ parameters.name }} + displayName: E2E - ${{ parameters.displayName }} + dependsOn: + - setup + - publish + jobs: + - job: ${{ parameters.name }} + displayName: DualStack Overlay Test Suite - (${{ parameters.name }}) + timeoutInMinutes: 120 + pool: + name: $(BUILD_POOL_NAME_DEFAULT) + demands: + - Role -equals $(CUSTOM_E2E_ROLE) + variables: + GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path + GOBIN: "$(GOPATH)/bin" # Go binaries path + modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking" + steps: + - template: dualstackoverlay-e2e-step-template.yaml + parameters: + name: ${{ parameters.name }} + testDropgz: ${{ parameters.testDropgz }} + clusterName: ${{ parameters.clusterName }} + \ No newline at end of file diff --git a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml new file mode 100644 index 0000000000..72f85b8217 --- /dev/null +++ b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml @@ -0,0 +1,76 @@ +parameters: + name: "" + testDropgz: "" + clusterName: "" + +steps: + - bash: | + echo $UID + sudo rm -rf $(System.DefaultWorkingDirectory)/* + displayName: "Set up OS environment" + - checkout: self + + - bash: | + go version + go env + mkdir -p '$(GOBIN)' + mkdir -p '$(GOPATH)/pkg' + mkdir -p '$(modulePath)' + echo '##vso[task.prependpath]$(GOBIN)' + echo '##vso[task.prependpath]$(GOROOT)/bin' + name: "GoEnv" + displayName: "Set up the Go environment" + - task: AzureCLI@2 + inputs: + azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + echo "Check az version" + az version + echo "Install az cli extension preview" + az extension add --name aks-preview + mkdir -p ~/.kube/ + echo "Create AKS DualStack Overlay cluster" + make -C ./hack/swift azcfg AZCLI=az REGION=$(REGION_OVERLAY_CLUSTER_TEST) + make -C ./hack/swift dualstack-overlay-up AZCLI=az REGION=$(REGION_OVERLAY_CLUSTER_TEST) VM_SIZE=$(VM_SIZE) NODE_COUNT=$(NODE_COUNT) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) + echo "Dualstack Overlay Cluster is successfully created" + displayName: Create DualStackOverlay cluster + condition: succeeded() + + - script: | + ls -lah + pwd + echo "installing kubectl" + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl + kubectl cluster-info + kubectl get node + kubectl get po -owide -A + name: "installKubectl" + displayName: "Install kubectl on AKS dualstack overlay cluster" + + - script: | + make -C ./hack/swift dualstack-windows-up AZCLI=az VM_SIZE=$(VM_SIZE) NODE_COUNT=$(NODE_COUNT) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) + echo "Windows node are successfully added to Dualstack Overlay Cluster" + kubectl cluster-info + kubectl get node + kubectl get po -owide -A + name: "Add_Windows_Node" + displayName: "Add windows node on DualStackOverlay cluster" + + - task: AzureCLI@2 + inputs: + azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + echo "Deleting cluster" + make -C ./hack/swift azcfg AZCLI=az + make -C ./hack/swift down SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision) + echo "Cluster and resources down" + name: "CleanupDualStackOverlaycluster" + displayName: "Cleanup DualStack Overlay Cluster" + condition: always() \ No newline at end of file diff --git a/hack/aks/Makefile b/hack/aks/Makefile index b16019b1dc..d6badaefb5 100644 --- a/hack/aks/Makefile +++ b/hack/aks/Makefile @@ -211,6 +211,31 @@ windows-cniv1-up: rg-up overlay-net-up ## Bring up a Windows CNIv1 cluster @$(MAKE) set-kubeconf +dualstack-overlay-up: rg-up overlay-net-up ## Brings up an dualstack Overlay cluster with Linux node only + $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ + --kubernetes-version 1.26.3 \ + --node-count $(NODE_COUNT) \ + --node-vm-size $(VM_SIZE) \ + --network-plugin azure \ + --ip-families ipv4,ipv6 \ + --network-plugin-mode overlay \ + --subscription $(SUB) \ + --no-ssh-key \ + --yes + @$(MAKE) set-kubeconf + +dualstack-windows-up: ## Brings up windows nodes on dualstack overlay cluster + $(AZCLI) aks nodepool add -g $(GROUP) -n npwin \ + --cluster-name $(CLUSTER) \ + --node-count $(NODE_COUNT) \ + --node-vm-size $(VM_SIZE) \ + --os-type Windows \ + --os-sku Windows2022 \ + --subscription $(SUB) \ + --no-ssh-key \ + --yes + @$(MAKE) set-kubeconf + down: ## Delete the cluster $(AZCLI) aks delete -g $(GROUP) -n $(CLUSTER) --yes @$(MAKE) unset-kubeconf diff --git a/hack/aks/README.md b/hack/aks/README.md index 3a4e80e4f0..ce29df2927 100644 --- a/hack/aks/README.md +++ b/hack/aks/README.md @@ -21,14 +21,16 @@ SWIFT Infra net-up Create required swift vnet/subnets AKS Clusters - byocni-up Alias to swift-byocni-up - cilium-up Alias to swift-cilium-up - up Alias to swift-up - overlay-up Brings up an Overlay AzCNI cluster - swift-byocni-up Bring up a SWIFT BYO CNI cluster - swift-cilium-up Bring up a SWIFT Cilium cluster - swift-up Bring up a SWIFT AzCNI cluster - windows-cniv1-up Bring up a Windows AzCNIv1 cluster - down Delete the cluster - vmss-restart Restart the nodes of the cluster + byocni-up Alias to swift-byocni-up + cilium-up Alias to swift-cilium-up + up Alias to swift-up + overlay-up Brings up an Overlay AzCNI cluster + swift-byocni-up Bring up a SWIFT BYO CNI cluster + swift-cilium-up Bring up a SWIFT Cilium cluster + swift-up Bring up a SWIFT AzCNI cluster + windows-cniv1-up Bring up a Windows AzCNIv1 cluster + dualstack-overlay-up Brings up an dualstack overlay cluster + dualstack-windows-up Brings up windows nodes on dualstack overlay cluster + down Delete the cluster + vmss-restart Restart the nodes of the cluster ``` From 0ca9a4be37693dfad19f10e0a864714c3684dea3 Mon Sep 17 00:00:00 2001 From: Paul Yu <129891899+paulyufan2@users.noreply.github.com> Date: Thu, 29 Jun 2023 12:15:34 -0400 Subject: [PATCH 15/45] fix conflicts --- test/integration/manifests/load/privileged-daemonset.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/integration/manifests/load/privileged-daemonset.yaml b/test/integration/manifests/load/privileged-daemonset.yaml index 77244f030a..519c81d3f2 100644 --- a/test/integration/manifests/load/privileged-daemonset.yaml +++ b/test/integration/manifests/load/privileged-daemonset.yaml @@ -33,11 +33,7 @@ spec: volumes: - name: azure-cns hostPath: -<<<<<<< HEAD path: /var/run/azure-network -======= - path: /var/run/azure-cns ->>>>>>> d691eef5 (fix conflicts) - name: azure-cns-noncilium hostPath: path: /var/run From 70f355da906f005d62f99f2184cd7e93e88fbb5c Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Fri, 30 Jun 2023 11:28:44 -0400 Subject: [PATCH 16/45] add windows properties check --- .../manifests/noop-deployment-windows.yaml | 2 +- test/validate/linux_validate.go | 24 ++++----- test/validate/windows_validate.go | 51 ++++++++++++++++--- 3 files changed, 56 insertions(+), 21 deletions(-) diff --git a/test/integration/manifests/noop-deployment-windows.yaml b/test/integration/manifests/noop-deployment-windows.yaml index 96619555c9..78467dffa5 100644 --- a/test/integration/manifests/noop-deployment-windows.yaml +++ b/test/integration/manifests/noop-deployment-windows.yaml @@ -16,7 +16,7 @@ spec: spec: containers: - name: noop - image: mcr.microsoft.com/windows/nanoserver:ltsc2022 + image: mcr.microsoft.com/dotnet/framework/samples:aspnetapp ports: - containerPort: 80 nodeSelector: diff --git a/test/validate/linux_validate.go b/test/validate/linux_validate.go index e8d68bd717..e38e6531e8 100644 --- a/test/validate/linux_validate.go +++ b/test/validate/linux_validate.go @@ -74,26 +74,26 @@ type Address struct { // parse azure-vnet.json // azure cni manages endpoint state type AzureCniState struct { - AzureCniState Network `json:"Network"` + AzureCniState AzureVnetNetwork `json:"Network"` } -type Network struct { +type AzureVnetNetwork struct { Version string `json:"Version"` TimeStamp string `json:"TimeStamp"` ExternalInterfaces map[string]InterfaceInfo `json:"ExternalInterfaces"` // key: interface name; value: Interface Info } type InterfaceInfo struct { - Name string `json:"Name"` - Networks map[string]NetworkInfo `json:"Networks"` // key: networkName, value: NetworkInfo + Name string `json:"Name"` + Networks map[string]AzureVnetNetworkInfo `json:"Networks"` // key: networkName, value: AzureVnetNetworkInfo } type AzureVnetInfo struct { Name string - Networks map[string]NetworkInfo // key: network name, value: NetworkInfo + Networks map[string]AzureVnetNetworkInfo // key: network name, value: NetworkInfo } -type NetworkInfo struct { +type AzureVnetNetworkInfo struct { ID string Mode string Subnets []Subnet @@ -158,7 +158,7 @@ func (l *LinuxClient) CreateClient(ctx context.Context, clienset *kubernetes.Cli func (v *LinuxValidator) ValidateStateFile() error { checkSet := make(map[string][]check) // key is cni type, value is a list of check - // TODO: add cniv1 when adding related test cases + // TODO: add cniv1 when adding Linux related test cases checkSet["cilium"] = []check{ {"cns", cnsStateFileIps, cnsLabelSelector, privilegedNamespace, cnsStateFileCmd}, {"cilium", ciliumStateFileIps, ciliumLabelSelector, privilegedNamespace, ciliumStateFileCmd}, @@ -167,7 +167,7 @@ func (v *LinuxValidator) ValidateStateFile() error { checkSet["cniv2"] = []check{ {"cns cache", cnsCacheStateFileIps, cnsLabelSelector, privilegedNamespace, cnsLocalCacheCmd}, - {"azure cns noncilium", azureCNSNonCiliumStateFileIPs, privilegedLabelSelector, privilegedNamespace, azureCnsStateFileCmd}, + {"azure dualstackoverlay", azureDualStackStateFileIPs, privilegedLabelSelector, privilegedNamespace, azureCnsStateFileCmd}, } for _, check := range checkSet[v.cni] { @@ -225,15 +225,15 @@ func cnsStateFileIps(result []byte) (map[string]string, error) { return cnsPodIps, nil } -func azureCNSNonCiliumStateFileIPs(result []byte) (map[string]string, error) { - var azureCNSNonCiliumResult AzureCniState - err := json.Unmarshal(result, &azureCNSNonCiliumResult) +func azureDualStackStateFileIPs(result []byte) (map[string]string, error) { + var azureDualStackResult AzureCniState + err := json.Unmarshal(result, &azureDualStackResult) if err != nil { return nil, errors.Wrapf(err, "failed to unmarshal azure cni endpoint list") } azureCnsPodIps := make(map[string]string) - for _, v := range azureCNSNonCiliumResult.AzureCniState.ExternalInterfaces { + for _, v := range azureDualStackResult.AzureCniState.ExternalInterfaces { for _, networks := range v.Networks { for _, ip := range networks.Endpoints { pod := ip.PODName diff --git a/test/validate/windows_validate.go b/test/validate/windows_validate.go index 9e54f61bef..4dce8cefa9 100644 --- a/test/validate/windows_validate.go +++ b/test/validate/windows_validate.go @@ -8,6 +8,7 @@ import ( k8sutils "github.com/Azure/azure-container-networking/test/internal/k8sutils" "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" ) @@ -106,24 +107,25 @@ func (w *WindowsClient) CreateClient(ctx context.Context, clienset *kubernetes.C } func (v *WindowsValidator) ValidateStateFile() error { - checks := []struct { - name string - stateFileIps func([]byte) (map[string]string, error) - podLabelSelector string - podNamespace string - cmd []string - }{ + checkSet := make(map[string][]check) // key is cni type, value is a list of check + + checkSet["cniv1"] = []check{ {"hns", hnsStateFileIps, privilegedLabelSelector, privilegedNamespace, hnsEndPpointCmd}, {"azure-vnet", azureVnetIps, privilegedLabelSelector, privilegedNamespace, azureVnetCmd}, {"azure-vnet-ipam", azureVnetIpamIps, privilegedLabelSelector, privilegedNamespace, azureVnetIpamCmd}, } - for _, check := range checks { + checkSet["cniv2"] = []check{ + {"azure-vnet", azureVnetIps, privilegedLabelSelector, privilegedNamespace, azureVnetCmd}, + } + + for _, check := range checkSet[v.cni] { err := v.validate(check.stateFileIps, check.cmd, check.name, check.podNamespace, check.podLabelSelector) if err != nil { return err } } + return nil } @@ -140,6 +142,7 @@ func hnsStateFileIps(result []byte) (map[string]string, error) { hnsPodIps[v.IPAddress.String()] = v.MacAddress } } + return hnsPodIps, nil } @@ -155,11 +158,13 @@ func azureVnetIps(result []byte) (map[string]string, error) { for _, v := range v.Networks { for _, e := range v.Endpoints { for _, v := range e.IPAddresses { + // collect both ipv4 and ipv6 addresses azureVnetPodIps[v.IP.String()] = e.IfName } } } } + return azureVnetPodIps, nil } @@ -226,3 +231,33 @@ func (v *WindowsValidator) validate(stateFileIps stateFileIpsFunc, cmd []string, func (v *WindowsValidator) ValidateRestartNetwork() error { return nil } + +func (v *WindowsValidator) ValidateDualStackNodeProperties() error { + log.Print("Validating Dualstack Overlay Windows Node properties") + nodes, err := k8sutils.GetNodeListByLabelSelector(v.ctx, v.clientset, windowsNodeSelector) + if err != nil { + return errors.Wrapf(err, "failed to get node list") + } + + for index := range nodes.Items { + nodeName := nodes.Items[index].ObjectMeta.Name + // check node status + nodeConditions := nodes.Items[index].Status.Conditions + if nodeConditions[len(nodeConditions)-1].Type != corev1.NodeReady { + return errors.Wrapf(err, "node %s status is not ready", nodeName) + } + + // get node labels + nodeLabels := nodes.Items[index].ObjectMeta.GetLabels() + for key := range nodeLabels { + if value, ok := dualstackoverlaynodelabel[key]; ok { + log.Printf("label %s is correctly shown on the node %+v", key, nodeName) + if value != overlayClusterLabelName { + return errors.Wrapf(err, "node %s overlay label name is wrong", nodeName) + } + } + } + } + + return nil +} From 5f55aab57ae26c7dc0f463fef7992be9eef6675c Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Fri, 30 Jun 2023 15:08:39 -0400 Subject: [PATCH 17/45] add windows dualstack hns test cases --- test/validate/windows_validate.go | 65 +++++++++++++++++++++++++++++-- 1 file changed, 61 insertions(+), 4 deletions(-) diff --git a/test/validate/windows_validate.go b/test/validate/windows_validate.go index 4dce8cefa9..b7dafa3032 100644 --- a/test/validate/windows_validate.go +++ b/test/validate/windows_validate.go @@ -19,7 +19,8 @@ const ( ) var ( - hnsEndPpointCmd = []string{"powershell", "-c", "Get-HnsEndpoint | ConvertTo-Json"} + hnsEndPointCmd = []string{"powershell", "-c", "Get-HnsEndpoint | ConvertTo-Json"} + hnsNetworkCmd = []string{"powershell", "-c", "Get-HnsNetwork | ConvertTo-Json"} azureVnetCmd = []string{"powershell", "-c", "cat ../../k/azure-vnet.json"} azureVnetIpamCmd = []string{"powershell", "-c", "cat ../../k/azure-vnet-ipam.json"} ) @@ -79,6 +80,14 @@ type AddressRecord struct { InUse bool } +type HNSNetwork struct { + Name string `json:"Name"` + IPv6 bool `json:"IPv6"` + ManagementIP string `json:"ManagementIP"` + ManagementIPv6 string `json:"ManagementIPv6"` + State int `json:"State"` +} + func (w *WindowsClient) CreateClient(ctx context.Context, clienset *kubernetes.Clientset, config *rest.Config, namespace, cni string, restartCase bool) IValidator { // deploy privileged pod privilegedDaemonSet, err := k8sutils.MustParseDaemonSet(privilegedWindowsDaemonSetPath) @@ -110,7 +119,7 @@ func (v *WindowsValidator) ValidateStateFile() error { checkSet := make(map[string][]check) // key is cni type, value is a list of check checkSet["cniv1"] = []check{ - {"hns", hnsStateFileIps, privilegedLabelSelector, privilegedNamespace, hnsEndPpointCmd}, + {"hns", hnsStateFileIps, privilegedLabelSelector, privilegedNamespace, hnsEndPointCmd}, {"azure-vnet", azureVnetIps, privilegedLabelSelector, privilegedNamespace, azureVnetCmd}, {"azure-vnet-ipam", azureVnetIpamIps, privilegedLabelSelector, privilegedNamespace, azureVnetIpamCmd}, } @@ -119,8 +128,9 @@ func (v *WindowsValidator) ValidateStateFile() error { {"azure-vnet", azureVnetIps, privilegedLabelSelector, privilegedNamespace, azureVnetCmd}, } + // this is checking all IPs of the pods with the statefile for _, check := range checkSet[v.cni] { - err := v.validate(check.stateFileIps, check.cmd, check.name, check.podNamespace, check.podLabelSelector) + err := v.validateIPs(check.stateFileIps, check.cmd, check.name, check.podNamespace, check.podLabelSelector) if err != nil { return err } @@ -146,6 +156,17 @@ func hnsStateFileIps(result []byte) (map[string]string, error) { return hnsPodIps, nil } +// return windows HNS network state +func hnsNetworkState(result []byte) ([]HNSNetwork, error) { + var hnsNetworkResult []HNSNetwork + err := json.Unmarshal(result, &hnsNetworkResult) + if err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal hns network list") + } + + return hnsNetworkResult, nil +} + func azureVnetIps(result []byte) (map[string]string, error) { var azureVnetResult AzureVnet err := json.Unmarshal(result, &azureVnetResult) @@ -189,7 +210,7 @@ func azureVnetIpamIps(result []byte) (map[string]string, error) { return azureVnetIpamPodIps, nil } -func (v *WindowsValidator) validate(stateFileIps stateFileIpsFunc, cmd []string, checkType, namespace, labelSelector string) error { +func (v *WindowsValidator) validateIPs(stateFileIps stateFileIpsFunc, cmd []string, checkType, namespace, labelSelector string) error { log.Println("Validating ", checkType, " state file") nodes, err := k8sutils.GetNodeListByLabelSelector(v.ctx, v.clientset, windowsNodeSelector) if err != nil { @@ -257,6 +278,42 @@ func (v *WindowsValidator) ValidateDualStackNodeProperties() error { } } } + + // check windows HNS network state + pod, err := k8sutils.GetPodsByNode(v.ctx, v.clientset, privilegedNamespace, privilegedLabelSelector, nodes.Items[index].Name) + if err != nil { + return errors.Wrapf(err, "failed to get privileged pod") + } + + podName := pod.Items[0].Name + // exec into the pod to get the state file + result, err := k8sutils.ExecCmdOnPod(v.ctx, v.clientset, privilegedNamespace, podName, hnsNetworkCmd, v.config) + if err != nil { + return errors.Wrapf(err, "failed to exec into privileged pod") + } + + hnsNetwork, err := hnsNetworkState(result) + if err != nil { + return errors.Wrapf(err, "failed to unmarshal hns network list") + } + + if len(hnsNetwork) == 0 { //nolint + return errors.Wrapf(err, "windows node does not have any HNS network") + } else if len(hnsNetwork) == 1 { //nolint + return errors.Wrapf(err, "HNS default ext network or azure network does not exist") + } else { + for _, network := range hnsNetwork { + if !network.IPv6 { + return errors.Wrapf(err, "windows HNS network IPv6 flag is not set correctly") + } + if network.State != 1 { + return errors.Wrapf(err, "windows HNS network state is not correct") + } + if network.ManagementIPv6 == "" || network.ManagementIP == "" { + return errors.Wrapf(err, "windows HNS network is missing ipv4 or ipv6 management IP") + } + } + } } return nil From e0c112b506a4304e6fab32a6dc56be1f7c5943aa Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Wed, 5 Jul 2023 11:18:40 -0400 Subject: [PATCH 18/45] dualstack overlay ipv6 windows connectivity tests --- .../datapath/datapath_windows_test.go | 21 ++++++--- test/internal/datapath/datapath_win.go | 43 ++++++++++++++++++- 2 files changed, 56 insertions(+), 8 deletions(-) diff --git a/test/integration/datapath/datapath_windows_test.go b/test/integration/datapath/datapath_windows_test.go index 054a60bb98..f6f42713b0 100644 --- a/test/integration/datapath/datapath_windows_test.go +++ b/test/integration/datapath/datapath_windows_test.go @@ -6,6 +6,7 @@ import ( "context" "flag" "fmt" + "net" "testing" "github.com/Azure/azure-container-networking/test/internal/datapath" @@ -36,9 +37,9 @@ k8s cluster with a windows nodepool consisting of at least 2 windows nodes. -nodepoolSelector="yournodepoolname" To run the test use one of the following commands: -go test -count=1 test/integration/datapath/datapath_win_test.go -timeout 3m -tags connection -run ^TestDatapathWin$ -tags=connection +go test -count=1 test/integration/datapath/datapath_windows_test.go -timeout 3m -tags connection -run ^TestDatapathWin$ -tags=connection or -go test -count=1 test/integration/datapath/datapath_win_test.go -timeout 3m -tags connection -run ^TestDatapathWin$ -podName=acnpod -nodepoolSelector=npwina -tags=connection +go test -count=1 test/integration/datapath/datapath_windows_test.go -timeout 3m -tags connection -run ^TestDatapathWin$ -podName=acnpod -nodepoolSelector=npwina -tags=connection This test checks pod to pod, pod to node, and pod to internet for datapath connectivity. @@ -110,7 +111,7 @@ func TestDatapathWin(t *testing.T) { require.NoError(t, err) } } - t.Log("Checking Windows test environment ") + t.Log("Checking Windows test environment") for _, node := range nodes.Items { pods, err := k8sutils.GetPodsByNode(ctx, clientset, *podNamespace, podLabelSelector, node.Name) @@ -129,18 +130,26 @@ func TestDatapathWin(t *testing.T) { for _, node := range nodes.Items { t.Log("Windows ping tests (1)") nodeIP := "" + nodeIPv6 := "" for _, address := range node.Status.Addresses { if address.Type == "InternalIP" { nodeIP = address.Address - // Multiple addresses exist, break once Internal IP found. - // Cannot call directly - break + if net.ParseIP(address.Address).To16() != nil { + nodeIPv6 = address.Address + } } } err := datapath.WindowsPodToNode(ctx, clientset, node.Name, nodeIP, *podNamespace, podLabelSelector, restConfig) require.NoError(t, err, "Windows pod to node, ping test failed with: %+v", err) t.Logf("Windows pod to node, passed for node: %s", node.Name) + + // windows ipv6 connectivity + if nodeIPv6 != "" { + err = datapath.WindowsPodToNode(ctx, clientset, node.Name, nodeIPv6, *podNamespace, podLabelSelector, restConfig) + require.NoError(t, err, "Windows pod to node, ipv6 ping test failed with: %+v", err) + t.Logf("Windows pod to node via ipv6, passed for node: %s", node.Name) + } } }) diff --git a/test/internal/datapath/datapath_win.go b/test/internal/datapath/datapath_win.go index 54a317760b..a8d77cfe66 100644 --- a/test/internal/datapath/datapath_win.go +++ b/test/internal/datapath/datapath_win.go @@ -3,6 +3,7 @@ package datapath import ( "context" "fmt" + "net" "strings" "github.com/Azure/azure-container-networking/test/internal/k8sutils" @@ -48,8 +49,28 @@ func WindowsPodToPodPingTestSameNode(ctx context.Context, clientset *kubernetes. } logrus.Infof("Second pod: %v %v", secondPod.Name, secondPod.Status.PodIP) + // ipv4 ping test // Ping the second pod from the first pod - return podTest(ctx, clientset, firstPod, []string{"ping", secondPod.Status.PodIP}, rc, pingPassedWindows) + resultOne := podTest(ctx, clientset, firstPod, []string{"ping", secondPod.Status.PodIP}, rc, pingPassedWindows) + if resultOne != nil { + return resultOne + } + + // ipv6 ping test + // ipv6 Ping the second pod from the first pod + if len(secondPod.Status.PodIPs) > 1 { + for _, ip := range secondPod.Status.PodIPs { + if net.ParseIP(ip.IP).To16() != nil { + secondPodIPv6 := ip.IP + resultTwo := podTest(ctx, clientset, firstPod, []string{"ping", secondPodIPv6}, rc, pingPassedWindows) + if resultTwo != nil { + return resultTwo + } + } + } + } + + return nil } func WindowsPodToPodPingTestDiffNode(ctx context.Context, clientset *kubernetes.Clientset, nodeName1, nodeName2, podNamespace, labelSelector string, rc *restclient.Config) error { @@ -80,7 +101,24 @@ func WindowsPodToPodPingTestDiffNode(ctx context.Context, clientset *kubernetes. logrus.Infof("Second pod: %v %v", secondPod.Name, secondPod.Status.PodIP) // Ping the second pod from the first pod located on different nodes - return podTest(ctx, clientset, firstPod, []string{"ping", secondPod.Status.PodIP}, rc, pingPassedWindows) + resultOne := podTest(ctx, clientset, firstPod, []string{"ping", secondPod.Status.PodIP}, rc, pingPassedWindows) + if resultOne != nil { + return resultOne + } + + if len(secondPod.Status.PodIPs) > 1 { + for _, ip := range secondPod.Status.PodIPs { + if net.ParseIP(ip.IP).To16() != nil { + secondPodIPv6 := ip.IP + resultTwo := podTest(ctx, clientset, firstPod, []string{"ping ", secondPodIPv6}, rc, pingPassedWindows) + if resultTwo != nil { + return resultTwo + } + } + } + } + + return nil } func WindowsPodToNode(ctx context.Context, clientset *kubernetes.Clientset, nodeName, nodeIP, podNamespace, labelSelector string, rc *restclient.Config) error { @@ -108,6 +146,7 @@ func WindowsPodToNode(ctx context.Context, clientset *kubernetes.Clientset, node logrus.Infof("Second pod: %v", secondPod.Name) // Ping from pod to node + logrus.Infof("Node IP: %s", nodeIP) resultOne := podTest(ctx, clientset, firstPod, []string{"ping", nodeIP}, rc, pingPassedWindows) resultTwo := podTest(ctx, clientset, secondPod, []string{"ping", nodeIP}, rc, pingPassedWindows) From 7994f7cb4526469dff68ac98f1c07876089999cb Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Wed, 5 Jul 2023 11:20:32 -0400 Subject: [PATCH 19/45] fix a minor issue --- test/internal/datapath/datapath_win.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/internal/datapath/datapath_win.go b/test/internal/datapath/datapath_win.go index a8d77cfe66..f026833570 100644 --- a/test/internal/datapath/datapath_win.go +++ b/test/internal/datapath/datapath_win.go @@ -61,8 +61,7 @@ func WindowsPodToPodPingTestSameNode(ctx context.Context, clientset *kubernetes. if len(secondPod.Status.PodIPs) > 1 { for _, ip := range secondPod.Status.PodIPs { if net.ParseIP(ip.IP).To16() != nil { - secondPodIPv6 := ip.IP - resultTwo := podTest(ctx, clientset, firstPod, []string{"ping", secondPodIPv6}, rc, pingPassedWindows) + resultTwo := podTest(ctx, clientset, firstPod, []string{"ping", ip.IP}, rc, pingPassedWindows) if resultTwo != nil { return resultTwo } @@ -109,8 +108,7 @@ func WindowsPodToPodPingTestDiffNode(ctx context.Context, clientset *kubernetes. if len(secondPod.Status.PodIPs) > 1 { for _, ip := range secondPod.Status.PodIPs { if net.ParseIP(ip.IP).To16() != nil { - secondPodIPv6 := ip.IP - resultTwo := podTest(ctx, clientset, firstPod, []string{"ping ", secondPodIPv6}, rc, pingPassedWindows) + resultTwo := podTest(ctx, clientset, firstPod, []string{"ping ", ip.IP}, rc, pingPassedWindows) if resultTwo != nil { return resultTwo } From ef93c7934a559ff7e2a943cedaf118ee3592e8ec Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Wed, 5 Jul 2023 14:09:28 -0400 Subject: [PATCH 20/45] add dualstack test cases to pipeline --- .../dualstackoverlay-e2e-step-template.yaml | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml index 72f85b8217..9ce5ce749c 100644 --- a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml @@ -60,6 +60,32 @@ steps: name: "Add_Windows_Node" displayName: "Add windows node on DualStackOverlay cluster" + - script: | + echo "DualStack Overlay Linux control plane Load test" + cd test/integration/load + go test -timeout 30m -tags load -run ^TestLoad$ -tags=load + echo "DualStack Overlay Linux control plane CNS validation test" + go test -timeout 30m -tags load -cni cniv2 -run ^TestValidateState$ -tags=load + echo "DualStack Overlay Linux control plane Node properties test" + go test -timeout 30m -tags load -run ^TestDualStackProperties$ -tags=load + echo "DualStack Overlay Linux datapath test" + go test -count=1 test/integration/datapath/datapath_linux_test.go -timeout 1m -tags connection -run ^TestDatapathLinux$ -tags=connection,integration + name: "DualStack_Overlay_Linux_tests" + displayName: "DualStack Overlay Linux Tests" + + - script: | + echo "DualStack Overlay Windows control plane Load test" + cd test/integration/load + go test -timeout 30m -tags load -run ^TestLoad$ -tags=load -os=windows + echo "DualStack Overlay Windows control plane CNS validation test" + go test -timeout 30m -tags load -run ^TestDualStackProperties$ -tags=load -os=windows -cni cniv2 + echo "DualStack Overlay Windows control plane Node properties test" + go test -timeout 30m -tags load -cni cniv2 -run ^TestValidateState$ -tags=load -os=windows + echo "DualStack Overlay Windows datapath test" + go test -count=1 test/integration/datapath/datapath_windows_test.go -timeout 3m -tags connection -run ^TestDatapathWin$ -tags=connection + name: "DualStack_Overlay_Windows_tests" + displayName: "DualStack Overlay Windows Tests" + - task: AzureCLI@2 inputs: azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) From 8a31eb034f8707fa7f9fe7c2f0fb035e7854b7d1 Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Wed, 5 Jul 2023 14:19:09 -0400 Subject: [PATCH 21/45] remove an echo --- .../dualstack-overlay/dualstackoverlay-e2e-step-template.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml index 9ce5ce749c..9423809d17 100644 --- a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml @@ -61,7 +61,6 @@ steps: displayName: "Add windows node on DualStackOverlay cluster" - script: | - echo "DualStack Overlay Linux control plane Load test" cd test/integration/load go test -timeout 30m -tags load -run ^TestLoad$ -tags=load echo "DualStack Overlay Linux control plane CNS validation test" @@ -74,7 +73,6 @@ steps: displayName: "DualStack Overlay Linux Tests" - script: | - echo "DualStack Overlay Windows control plane Load test" cd test/integration/load go test -timeout 30m -tags load -run ^TestLoad$ -tags=load -os=windows echo "DualStack Overlay Windows control plane CNS validation test" From 20c0bd08b0009c5b8708bd43450031101460b487 Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Wed, 5 Jul 2023 19:35:42 -0400 Subject: [PATCH 22/45] correct aks path --- .../dualstackoverlay-e2e-step-template.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml index 9423809d17..42765247ea 100644 --- a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml @@ -33,8 +33,8 @@ steps: az extension add --name aks-preview mkdir -p ~/.kube/ echo "Create AKS DualStack Overlay cluster" - make -C ./hack/swift azcfg AZCLI=az REGION=$(REGION_OVERLAY_CLUSTER_TEST) - make -C ./hack/swift dualstack-overlay-up AZCLI=az REGION=$(REGION_OVERLAY_CLUSTER_TEST) VM_SIZE=$(VM_SIZE) NODE_COUNT=$(NODE_COUNT) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) + make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_OVERLAY_CLUSTER_TEST) + make -C ./hack/aks dualstack-overlay-up AZCLI=az REGION=$(REGION_OVERLAY_CLUSTER_TEST) VM_SIZE=$(VM_SIZE) NODE_COUNT=$(NODE_COUNT) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) echo "Dualstack Overlay Cluster is successfully created" displayName: Create DualStackOverlay cluster condition: succeeded() @@ -52,7 +52,7 @@ steps: displayName: "Install kubectl on AKS dualstack overlay cluster" - script: | - make -C ./hack/swift dualstack-windows-up AZCLI=az VM_SIZE=$(VM_SIZE) NODE_COUNT=$(NODE_COUNT) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) + make -C ./hack/aks dualstack-windows-up AZCLI=az VM_SIZE=$(VM_SIZE) NODE_COUNT=$(NODE_COUNT) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) echo "Windows node are successfully added to Dualstack Overlay Cluster" kubectl cluster-info kubectl get node @@ -92,8 +92,8 @@ steps: addSpnToEnvironment: true inlineScript: | echo "Deleting cluster" - make -C ./hack/swift azcfg AZCLI=az - make -C ./hack/swift down SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision) + make -C ./hack/aks azcfg AZCLI=az + make -C ./hack/aks down SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision) echo "Cluster and resources down" name: "CleanupDualStackOverlaycluster" displayName: "Cleanup DualStack Overlay Cluster" From 743b5d0600bc64152f8f1b33a8b126879a03a378 Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Thu, 6 Jul 2023 15:55:27 -0400 Subject: [PATCH 23/45] fix minor issues --- .../dualstackoverlay-e2e-step-template.yaml | 9 ++++++--- hack/aks/Makefile | 1 - 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml index 42765247ea..ab52f40ffd 100644 --- a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml @@ -31,6 +31,7 @@ steps: az version echo "Install az cli extension preview" az extension add --name aks-preview + az extension update --name aks-preview mkdir -p ~/.kube/ echo "Create AKS DualStack Overlay cluster" make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_OVERLAY_CLUSTER_TEST) @@ -68,19 +69,21 @@ steps: echo "DualStack Overlay Linux control plane Node properties test" go test -timeout 30m -tags load -run ^TestDualStackProperties$ -tags=load echo "DualStack Overlay Linux datapath test" - go test -count=1 test/integration/datapath/datapath_linux_test.go -timeout 1m -tags connection -run ^TestDatapathLinux$ -tags=connection,integration + cd ../datapath + go test -count=1 datapath_linux_test.go -timeout 1m -tags connection -run ^TestDatapathLinux$ -tags=connection,integration name: "DualStack_Overlay_Linux_tests" displayName: "DualStack Overlay Linux Tests" - script: | - cd test/integration/load + cd ../load go test -timeout 30m -tags load -run ^TestLoad$ -tags=load -os=windows echo "DualStack Overlay Windows control plane CNS validation test" go test -timeout 30m -tags load -run ^TestDualStackProperties$ -tags=load -os=windows -cni cniv2 echo "DualStack Overlay Windows control plane Node properties test" go test -timeout 30m -tags load -cni cniv2 -run ^TestValidateState$ -tags=load -os=windows echo "DualStack Overlay Windows datapath test" - go test -count=1 test/integration/datapath/datapath_windows_test.go -timeout 3m -tags connection -run ^TestDatapathWin$ -tags=connection + cd ../datapath + go test -count=1 datapath_windows_test.go -timeout 3m -tags connection -run ^TestDatapathWin$ -tags=connection name: "DualStack_Overlay_Windows_tests" displayName: "DualStack Overlay Windows Tests" diff --git a/hack/aks/Makefile b/hack/aks/Makefile index d6badaefb5..4ccd267fbe 100644 --- a/hack/aks/Makefile +++ b/hack/aks/Makefile @@ -230,7 +230,6 @@ dualstack-windows-up: ## Brings up windows nodes on dualstack overlay cluster --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ --os-type Windows \ - --os-sku Windows2022 \ --subscription $(SUB) \ --no-ssh-key \ --yes From 97a1f1e6f618901546777277befd3e5cbf4ad952 Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Thu, 6 Jul 2023 16:13:17 -0400 Subject: [PATCH 24/45] remove yes --- hack/aks/Makefile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/hack/aks/Makefile b/hack/aks/Makefile index 4ccd267fbe..595aefcebd 100644 --- a/hack/aks/Makefile +++ b/hack/aks/Makefile @@ -230,9 +230,7 @@ dualstack-windows-up: ## Brings up windows nodes on dualstack overlay cluster --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ --os-type Windows \ - --subscription $(SUB) \ - --no-ssh-key \ - --yes + --subscription $(SUB) @$(MAKE) set-kubeconf down: ## Delete the cluster From 2723a46dbee28799057d66473dc6cff8e9c4e065 Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Thu, 6 Jul 2023 16:41:16 -0400 Subject: [PATCH 25/45] add AzureCLI for windows nodepool --- .../dualstackoverlay-e2e-step-template.yaml | 18 ++++++++++++------ hack/aks/Makefile | 5 ++++- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml index ab52f40ffd..785a5c675f 100644 --- a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml @@ -52,12 +52,18 @@ steps: name: "installKubectl" displayName: "Install kubectl on AKS dualstack overlay cluster" - - script: | - make -C ./hack/aks dualstack-windows-up AZCLI=az VM_SIZE=$(VM_SIZE) NODE_COUNT=$(NODE_COUNT) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) - echo "Windows node are successfully added to Dualstack Overlay Cluster" - kubectl cluster-info - kubectl get node - kubectl get po -owide -A + - task: AzureCLI@2 + inputs: + azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + make -C ./hack/aks dualstack-windows-up AZCLI=az VM_SIZE=$(VM_SIZE) NODE_COUNT=$(NODE_COUNT) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) + echo "Windows node are successfully added to Dualstack Overlay Cluster" + kubectl cluster-info + kubectl get node + kubectl get po -owide -A name: "Add_Windows_Node" displayName: "Add windows node on DualStackOverlay cluster" diff --git a/hack/aks/Makefile b/hack/aks/Makefile index 595aefcebd..d6badaefb5 100644 --- a/hack/aks/Makefile +++ b/hack/aks/Makefile @@ -230,7 +230,10 @@ dualstack-windows-up: ## Brings up windows nodes on dualstack overlay cluster --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ --os-type Windows \ - --subscription $(SUB) + --os-sku Windows2022 \ + --subscription $(SUB) \ + --no-ssh-key \ + --yes @$(MAKE) set-kubeconf down: ## Delete the cluster From 46c945c0df30ac04ef54d5365f7211a622f08c33 Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Thu, 6 Jul 2023 16:51:32 -0400 Subject: [PATCH 26/45] remove ssh key check --- hack/aks/Makefile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/hack/aks/Makefile b/hack/aks/Makefile index d6badaefb5..5b26337550 100644 --- a/hack/aks/Makefile +++ b/hack/aks/Makefile @@ -231,9 +231,7 @@ dualstack-windows-up: ## Brings up windows nodes on dualstack overlay cluster --node-vm-size $(VM_SIZE) \ --os-type Windows \ --os-sku Windows2022 \ - --subscription $(SUB) \ - --no-ssh-key \ - --yes + --subscription $(SUB) @$(MAKE) set-kubeconf down: ## Delete the cluster From 19bca7583f5afc02c4d0e66622cb0577e9d01fcb Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Thu, 6 Jul 2023 17:50:56 -0400 Subject: [PATCH 27/45] change the Linux test cases sequence --- .../dualstackoverlay-e2e-step-template.yaml | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml index 785a5c675f..8a985b7a35 100644 --- a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml @@ -52,6 +52,19 @@ steps: name: "installKubectl" displayName: "Install kubectl on AKS dualstack overlay cluster" + - script: | + cd test/integration/load + go test -timeout 30m -tags load -run ^TestLoad$ -tags=load + echo "DualStack Overlay Linux control plane CNS validation test" + go test -timeout 30m -tags load -cni cniv2 -run ^TestValidateState$ -tags=load + echo "DualStack Overlay Linux control plane Node properties test" + go test -timeout 30m -tags load -run ^TestDualStackProperties$ -tags=load + echo "DualStack Overlay Linux datapath test" + cd ../datapath + go test -count=1 datapath_linux_test.go -timeout 1m -tags connection -run ^TestDatapathLinux$ -tags=connection,integration + name: "DualStack_Overlay_Linux_tests" + displayName: "DualStack Overlay Linux Tests" + - task: AzureCLI@2 inputs: azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) @@ -67,19 +80,6 @@ steps: name: "Add_Windows_Node" displayName: "Add windows node on DualStackOverlay cluster" - - script: | - cd test/integration/load - go test -timeout 30m -tags load -run ^TestLoad$ -tags=load - echo "DualStack Overlay Linux control plane CNS validation test" - go test -timeout 30m -tags load -cni cniv2 -run ^TestValidateState$ -tags=load - echo "DualStack Overlay Linux control plane Node properties test" - go test -timeout 30m -tags load -run ^TestDualStackProperties$ -tags=load - echo "DualStack Overlay Linux datapath test" - cd ../datapath - go test -count=1 datapath_linux_test.go -timeout 1m -tags connection -run ^TestDatapathLinux$ -tags=connection,integration - name: "DualStack_Overlay_Linux_tests" - displayName: "DualStack Overlay Linux Tests" - - script: | cd ../load go test -timeout 30m -tags load -run ^TestLoad$ -tags=load -os=windows From 93b3e770a92900aef2e89f3a57b6e5d22c7eec1f Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Thu, 6 Jul 2023 21:10:25 -0400 Subject: [PATCH 28/45] add byo-cni --- .../dualstackoverlay-e2e-step-template.yaml | 5 +++-- hack/aks/Makefile | 12 ++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml index 8a985b7a35..989e136edd 100644 --- a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml @@ -64,7 +64,7 @@ steps: go test -count=1 datapath_linux_test.go -timeout 1m -tags connection -run ^TestDatapathLinux$ -tags=connection,integration name: "DualStack_Overlay_Linux_tests" displayName: "DualStack Overlay Linux Tests" - + - task: AzureCLI@2 inputs: azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) @@ -81,7 +81,8 @@ steps: displayName: "Add windows node on DualStackOverlay cluster" - script: | - cd ../load + pwd + cd test/integration/load go test -timeout 30m -tags load -run ^TestLoad$ -tags=load -os=windows echo "DualStack Overlay Windows control plane CNS validation test" go test -timeout 30m -tags load -run ^TestDualStackProperties$ -tags=load -os=windows -cni cniv2 diff --git a/hack/aks/Makefile b/hack/aks/Makefile index 5b26337550..a5ee01018f 100644 --- a/hack/aks/Makefile +++ b/hack/aks/Makefile @@ -224,6 +224,18 @@ dualstack-overlay-up: rg-up overlay-net-up ## Brings up an dualstack Overlay clu --yes @$(MAKE) set-kubeconf +dualstack-overlay-byocni-up: rg-up overlay-net-up ## Brings up an dualstack Overlay BYO CNI cluster + $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ + --node-count $(NODE_COUNT) \ + --node-vm-size $(VM_SIZE) \ + --network-plugin none \ + --network-plugin-mode overlay \ + --ip-families ipv4,ipv6 \ + --subscription $(SUB) \ + --no-ssh-key \ + --yes + @$(MAKE) set-kubeconf + dualstack-windows-up: ## Brings up windows nodes on dualstack overlay cluster $(AZCLI) aks nodepool add -g $(GROUP) -n npwin \ --cluster-name $(CLUSTER) \ From bb134ed6daa214327905eb924004c4c2aec9443a Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Thu, 6 Jul 2023 23:08:31 -0400 Subject: [PATCH 29/45] remove ns-load ns --- .../dualstack-overlay/dualstackoverlay-e2e-step-template.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml index 989e136edd..60155ce673 100644 --- a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml @@ -83,6 +83,8 @@ steps: - script: | pwd cd test/integration/load + echo "Delete Linux load-test namespace" + kubectl delete ns load-test go test -timeout 30m -tags load -run ^TestLoad$ -tags=load -os=windows echo "DualStack Overlay Windows control plane CNS validation test" go test -timeout 30m -tags load -run ^TestDualStackProperties$ -tags=load -os=windows -cni cniv2 From e75c993245f652923a209545b6354618af528d92 Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Thu, 6 Jul 2023 23:11:25 -0400 Subject: [PATCH 30/45] remove ns-load ns --- .../dualstack-overlay/dualstackoverlay-e2e-step-template.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml index 60155ce673..652bb2006a 100644 --- a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml @@ -62,6 +62,8 @@ steps: echo "DualStack Overlay Linux datapath test" cd ../datapath go test -count=1 datapath_linux_test.go -timeout 1m -tags connection -run ^TestDatapathLinux$ -tags=connection,integration + echo "Delete Linux load-test namespace" + kubectl delete ns load-test name: "DualStack_Overlay_Linux_tests" displayName: "DualStack Overlay Linux Tests" @@ -83,8 +85,6 @@ steps: - script: | pwd cd test/integration/load - echo "Delete Linux load-test namespace" - kubectl delete ns load-test go test -timeout 30m -tags load -run ^TestLoad$ -tags=load -os=windows echo "DualStack Overlay Windows control plane CNS validation test" go test -timeout 30m -tags load -run ^TestDualStackProperties$ -tags=load -os=windows -cni cniv2 From 943926bfc7cc63b649d84c47853dba493d7633c7 Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Fri, 7 Jul 2023 13:49:50 -0400 Subject: [PATCH 31/45] fix deployment files --- test/integration/manifests/load/privileged-daemonset.yaml | 7 ++++++- test/integration/manifests/noop-deployment-windows.yaml | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/test/integration/manifests/load/privileged-daemonset.yaml b/test/integration/manifests/load/privileged-daemonset.yaml index 519c81d3f2..45398306d5 100644 --- a/test/integration/manifests/load/privileged-daemonset.yaml +++ b/test/integration/manifests/load/privileged-daemonset.yaml @@ -24,14 +24,19 @@ spec: privileged: true runAsUser: 0 volumeMounts: - - mountPath: /var/run/azure-network + - mountPath: /var/run/azure-cns name: azure-cns + - mountPath: /var/run/azure-network + name: azure-network - mountPath: /host name: host-root - mountPath: /var/run name: azure-cns-noncilium volumes: - name: azure-cns + hostPath: + path: /var/run/azure-cns + - name: azure-network hostPath: path: /var/run/azure-network - name: azure-cns-noncilium diff --git a/test/integration/manifests/noop-deployment-windows.yaml b/test/integration/manifests/noop-deployment-windows.yaml index 78467dffa5..3ca88524e5 100644 --- a/test/integration/manifests/noop-deployment-windows.yaml +++ b/test/integration/manifests/noop-deployment-windows.yaml @@ -16,8 +16,8 @@ spec: spec: containers: - name: noop - image: mcr.microsoft.com/dotnet/framework/samples:aspnetapp + image: mcr.microsoft.com/oss/kubernetes/pause:3.6-windows-ltsc2022-amd64 ports: - containerPort: 80 nodeSelector: - "kubernetes.io/os": windows + kubernetes.io/os: windows From 3e5eb23349ea1525091e672ebf94aa1561def220 Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Mon, 10 Jul 2023 17:31:45 -0400 Subject: [PATCH 32/45] windows ipv6 invoke-webrequest test case --- test/internal/datapath/datapath_win.go | 24 ++++++++++++++++++++++++ test/validate/windows_validate.go | 1 + 2 files changed, 25 insertions(+) diff --git a/test/internal/datapath/datapath_win.go b/test/internal/datapath/datapath_win.go index f026833570..4d792cbc89 100644 --- a/test/internal/datapath/datapath_win.go +++ b/test/internal/datapath/datapath_win.go @@ -15,6 +15,10 @@ import ( restclient "k8s.io/client-go/rest" ) +var ( + ipv6PrefixPolicy = []string{"powershell", "-c", "netsh interface ipv6 add prefixpolicy fd00::/8 3 1"} +) + func podTest(ctx context.Context, clientset *kubernetes.Clientset, srcPod *apiv1.Pod, cmd []string, rc *restclient.Config, passFunc func(string) error) error { logrus.Infof("podTest() - %v %v", srcPod.Name, cmd) output, err := k8sutils.ExecCmdOnPod(ctx, clientset, srcPod.Namespace, srcPod.Name, cmd, rc) @@ -195,6 +199,26 @@ func WindowsPodToInternet(ctx context.Context, clientset *kubernetes.Clientset, return resultTwo } + // test Invoke-WebRequest an URL by IPv6 address on one pod + // to Invoke-WebRequest a website by using IPv6 address, need to apply IPv6PrefixPolicy "netsh interface ipv6 add prefixpolicy fd00::/8 3 1" + // then PS C:\> Test-NetConnection www.bing.com + // RemoteAddress : 2620:1ec:c11::200 + if len(secondPod.Status.PodIPs) > 1 { + for _, ip := range secondPod.Status.PodIPs { + if net.ParseIP(ip.IP).To16() != nil { + _, err = k8sutils.ExecCmdOnPod(ctx, clientset, podNamespace, pods.Items[0].Name, ipv6PrefixPolicy, rc) + if err != nil { + return errors.Wrapf(err, "failed to exec command on windows pod") + } + + resultThree := podTest(ctx, clientset, secondPod, []string{"powershell", "Invoke-WebRequest", "www.bing.com", "-UseBasicParsing"}, rc, webRequestPassedWindows) + if resultThree != nil { + return resultThree + } + } + } + } + return nil } diff --git a/test/validate/windows_validate.go b/test/validate/windows_validate.go index b7dafa3032..0f0332f46b 100644 --- a/test/validate/windows_validate.go +++ b/test/validate/windows_validate.go @@ -23,6 +23,7 @@ var ( hnsNetworkCmd = []string{"powershell", "-c", "Get-HnsNetwork | ConvertTo-Json"} azureVnetCmd = []string{"powershell", "-c", "cat ../../k/azure-vnet.json"} azureVnetIpamCmd = []string{"powershell", "-c", "cat ../../k/azure-vnet-ipam.json"} + nsLookupCmd = []string{"powershell", "nslookup", "www.bing.com"} ) type WindowsClient struct{} From bec1fadac7fe91b575d7dd17f52a47b87989b19f Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Tue, 11 Jul 2023 09:29:08 -0400 Subject: [PATCH 33/45] fix issues --- hack/aks/Makefile | 1 + test/validate/windows_validate.go | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/aks/Makefile b/hack/aks/Makefile index a5ee01018f..b46e6b9865 100644 --- a/hack/aks/Makefile +++ b/hack/aks/Makefile @@ -226,6 +226,7 @@ dualstack-overlay-up: rg-up overlay-net-up ## Brings up an dualstack Overlay clu dualstack-overlay-byocni-up: rg-up overlay-net-up ## Brings up an dualstack Overlay BYO CNI cluster $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ + --kubernetes-version 1.26.3 \ --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ --network-plugin none \ diff --git a/test/validate/windows_validate.go b/test/validate/windows_validate.go index 0f0332f46b..b7dafa3032 100644 --- a/test/validate/windows_validate.go +++ b/test/validate/windows_validate.go @@ -23,7 +23,6 @@ var ( hnsNetworkCmd = []string{"powershell", "-c", "Get-HnsNetwork | ConvertTo-Json"} azureVnetCmd = []string{"powershell", "-c", "cat ../../k/azure-vnet.json"} azureVnetIpamCmd = []string{"powershell", "-c", "cat ../../k/azure-vnet-ipam.json"} - nsLookupCmd = []string{"powershell", "nslookup", "www.bing.com"} ) type WindowsClient struct{} From 975a46bb215de490fe703d73848a402f8c0b99c0 Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Mon, 17 Jul 2023 12:46:53 -0400 Subject: [PATCH 34/45] fix ipv6 command --- test/internal/datapath/datapath_win.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/internal/datapath/datapath_win.go b/test/internal/datapath/datapath_win.go index 4d792cbc89..4cc80667ab 100644 --- a/test/internal/datapath/datapath_win.go +++ b/test/internal/datapath/datapath_win.go @@ -16,7 +16,7 @@ import ( ) var ( - ipv6PrefixPolicy = []string{"powershell", "-c", "netsh interface ipv6 add prefixpolicy fd00::/8 3 1"} + ipv6PrefixPolicy = []string{"powershell", "netsh interface ipv6 add prefixpolicy fd00::/8 3 1"} ) func podTest(ctx context.Context, clientset *kubernetes.Clientset, srcPod *apiv1.Pod, cmd []string, rc *restclient.Config, passFunc func(string) error) error { From e7a7d7f30c23dfd508d13dc90f5eefa1ca0dd57e Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Mon, 17 Jul 2023 14:11:34 -0400 Subject: [PATCH 35/45] fix windows ipv6 command --- test/internal/datapath/datapath_win.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/internal/datapath/datapath_win.go b/test/internal/datapath/datapath_win.go index 4cc80667ab..a78bcf378d 100644 --- a/test/internal/datapath/datapath_win.go +++ b/test/internal/datapath/datapath_win.go @@ -16,7 +16,7 @@ import ( ) var ( - ipv6PrefixPolicy = []string{"powershell", "netsh interface ipv6 add prefixpolicy fd00::/8 3 1"} + ipv6PrefixPolicy = []string{"powershell", "-c", "netsh interface ipv6 add prefixpolicy fd00::/8 3 1"} ) func podTest(ctx context.Context, clientset *kubernetes.Clientset, srcPod *apiv1.Pod, cmd []string, rc *restclient.Config, passFunc func(string) error) error { @@ -205,6 +205,7 @@ func WindowsPodToInternet(ctx context.Context, clientset *kubernetes.Clientset, // RemoteAddress : 2620:1ec:c11::200 if len(secondPod.Status.PodIPs) > 1 { for _, ip := range secondPod.Status.PodIPs { + logrus.Infof("pods.Items[0].Name is %s", pods.Items[0].Name) if net.ParseIP(ip.IP).To16() != nil { _, err = k8sutils.ExecCmdOnPod(ctx, clientset, podNamespace, pods.Items[0].Name, ipv6PrefixPolicy, rc) if err != nil { From bf1be0ef3b3f6232e16a863947582161d79d7c5d Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Mon, 17 Jul 2023 17:15:56 -0400 Subject: [PATCH 36/45] test v6 curl --- test/internal/datapath/datapath_win.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/test/internal/datapath/datapath_win.go b/test/internal/datapath/datapath_win.go index a78bcf378d..72256e0ec5 100644 --- a/test/internal/datapath/datapath_win.go +++ b/test/internal/datapath/datapath_win.go @@ -16,7 +16,7 @@ import ( ) var ( - ipv6PrefixPolicy = []string{"powershell", "-c", "netsh interface ipv6 add prefixpolicy fd00::/8 3 1"} + ipv6PrefixPolicy = []string{"curl", "-6", "-I", "-v", "www.bing.com"} ) func podTest(ctx context.Context, clientset *kubernetes.Clientset, srcPod *apiv1.Pod, cmd []string, rc *restclient.Config, passFunc func(string) error) error { @@ -207,12 +207,7 @@ func WindowsPodToInternet(ctx context.Context, clientset *kubernetes.Clientset, for _, ip := range secondPod.Status.PodIPs { logrus.Infof("pods.Items[0].Name is %s", pods.Items[0].Name) if net.ParseIP(ip.IP).To16() != nil { - _, err = k8sutils.ExecCmdOnPod(ctx, clientset, podNamespace, pods.Items[0].Name, ipv6PrefixPolicy, rc) - if err != nil { - return errors.Wrapf(err, "failed to exec command on windows pod") - } - - resultThree := podTest(ctx, clientset, secondPod, []string{"powershell", "Invoke-WebRequest", "www.bing.com", "-UseBasicParsing"}, rc, webRequestPassedWindows) + resultThree := podTest(ctx, clientset, secondPod, ipv6PrefixPolicy, rc, webRequestPassedWindows) if resultThree != nil { return resultThree } From 878ed1cab1442cf312bce7a66424e2e17d4d0c0c Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Mon, 17 Jul 2023 19:21:14 -0400 Subject: [PATCH 37/45] install linux cns and cni --- .../dualstackoverlay-e2e-step-template.yaml | 10 +++++++++- test/internal/datapath/datapath_win.go | 10 ++++------ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml index 652bb2006a..e5fe2c1e7c 100644 --- a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml @@ -35,7 +35,7 @@ steps: mkdir -p ~/.kube/ echo "Create AKS DualStack Overlay cluster" make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_OVERLAY_CLUSTER_TEST) - make -C ./hack/aks dualstack-overlay-up AZCLI=az REGION=$(REGION_OVERLAY_CLUSTER_TEST) VM_SIZE=$(VM_SIZE) NODE_COUNT=$(NODE_COUNT) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) + make -C ./hack/aks dualstack-overlay-byocni-up AZCLI=az REGION=$(REGION_OVERLAY_CLUSTER_TEST) VM_SIZE=$(VM_SIZE) NODE_COUNT=$(NODE_COUNT) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) echo "Dualstack Overlay Cluster is successfully created" displayName: Create DualStackOverlay cluster condition: succeeded() @@ -52,6 +52,14 @@ steps: name: "installKubectl" displayName: "Install kubectl on AKS dualstack overlay cluster" + - script: | + echo "Start DualStack overlay E2E Tests" + kubectl get po -owide -A + sudo -E env "PATH=$PATH" make test-integration CNS_VERSION=$(make cns-version) CNI_DROPGZ_VERSION=$(make cni-dropgz-version) INSTALL_CNS=true INSTALL_AZILIUM=true TEST_DROPGZ=${{ parameters.testDropgz }} + retryCountOnTaskFailure: 3 + name: "dualStackOverlayTest" + displayName: "Run DualStack overlay E2E" + - script: | cd test/integration/load go test -timeout 30m -tags load -run ^TestLoad$ -tags=load diff --git a/test/internal/datapath/datapath_win.go b/test/internal/datapath/datapath_win.go index 72256e0ec5..d48084af70 100644 --- a/test/internal/datapath/datapath_win.go +++ b/test/internal/datapath/datapath_win.go @@ -15,9 +15,7 @@ import ( restclient "k8s.io/client-go/rest" ) -var ( - ipv6PrefixPolicy = []string{"curl", "-6", "-I", "-v", "www.bing.com"} -) +var ipv6PrefixPolicy = []string{"curl", "-6", "-I", "-v", "www.bing.com"} func podTest(ctx context.Context, clientset *kubernetes.Clientset, srcPod *apiv1.Pod, cmd []string, rc *restclient.Config, passFunc func(string) error) error { logrus.Infof("podTest() - %v %v", srcPod.Name, cmd) @@ -200,9 +198,9 @@ func WindowsPodToInternet(ctx context.Context, clientset *kubernetes.Clientset, } // test Invoke-WebRequest an URL by IPv6 address on one pod - // to Invoke-WebRequest a website by using IPv6 address, need to apply IPv6PrefixPolicy "netsh interface ipv6 add prefixpolicy fd00::/8 3 1" - // then PS C:\> Test-NetConnection www.bing.com - // RemoteAddress : 2620:1ec:c11::200 + // command is: C:\inetpub\wwwroot>curl -6 -I -v www.bing.com + // then return * Trying [2620:1ec:c11::200]:80... + // HTTP/1.1 200 OK if len(secondPod.Status.PodIPs) > 1 { for _, ip := range secondPod.Status.PodIPs { logrus.Infof("pods.Items[0].Name is %s", pods.Items[0].Name) From 2ec9b1952412fef9e7c7df50b6fdd04aec5a991a Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Tue, 18 Jul 2023 14:50:10 -0400 Subject: [PATCH 38/45] add aks-custom-headers --- hack/aks/Makefile | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/hack/aks/Makefile b/hack/aks/Makefile index 303b575fb9..8fa208975e 100644 --- a/hack/aks/Makefile +++ b/hack/aks/Makefile @@ -234,12 +234,26 @@ dualstack-overlay-byocni-up: rg-up overlay-net-up ## Brings up an dualstack Over --node-vm-size $(VM_SIZE) \ --network-plugin none \ --network-plugin-mode overlay \ + --aks-custom-headers AKSHTTPCustomFeatures=Microsoft.ContainerService/AzureOverlayDualStackPreview \ --ip-families ipv4,ipv6 \ --subscription $(SUB) \ --no-ssh-key \ --yes @$(MAKE) set-kubeconf +overlay-byocni-up: rg-up overlay-net-up ## Brings up an Overlay BYO CNI cluster + $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ + --node-count $(NODE_COUNT) \ + --node-vm-size $(VM_SIZE) \ + --load-balancer-sku basic \ + --network-plugin none \ + --network-plugin-mode overlay \ + --pod-cidr 192.168.0.0/16 \ + --vnet-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/nodenet \ + --no-ssh-key \ + --yes + @$(MAKE) set-kubeconf + dualstack-windows-up: ## Brings up windows nodes on dualstack overlay cluster $(AZCLI) aks nodepool add -g $(GROUP) -n npwin \ --cluster-name $(CLUSTER) \ From 6e5b69578cca606c4f9d21a92e156a83028cb81e Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Tue, 18 Jul 2023 14:56:17 -0400 Subject: [PATCH 39/45] add aks-custom-headers --- hack/aks/Makefile | 20 +++++++++----------- hack/aks/README.md | 26 ++++++++++++++------------ 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/hack/aks/Makefile b/hack/aks/Makefile index 8fa208975e..968c87b6d1 100644 --- a/hack/aks/Makefile +++ b/hack/aks/Makefile @@ -241,24 +241,22 @@ dualstack-overlay-byocni-up: rg-up overlay-net-up ## Brings up an dualstack Over --yes @$(MAKE) set-kubeconf -overlay-byocni-up: rg-up overlay-net-up ## Brings up an Overlay BYO CNI cluster - $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ +dualstack-windows-up: ## Brings up windows nodes on dualstack overlay cluster + $(AZCLI) aks nodepool add -g $(GROUP) -n npwin \ + --cluster-name $(CLUSTER) \ --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ - --load-balancer-sku basic \ - --network-plugin none \ - --network-plugin-mode overlay \ - --pod-cidr 192.168.0.0/16 \ - --vnet-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/nodenet \ - --no-ssh-key \ - --yes - @$(MAKE) set-kubeconf + --os-type Windows \ + --os-sku Windows2022 \ + --subscription $(SUB) + @$(MAKE) set-kubeconf -dualstack-windows-up: ## Brings up windows nodes on dualstack overlay cluster +dualstack-windows--byocni-up: ## Brings up windows nodes on dualstack overlay cluster without CNS and CNI installed $(AZCLI) aks nodepool add -g $(GROUP) -n npwin \ --cluster-name $(CLUSTER) \ --node-count $(NODE_COUNT) \ --node-vm-size $(VM_SIZE) \ + --network-plugin none \ --os-type Windows \ --os-sku Windows2022 \ --subscription $(SUB) diff --git a/hack/aks/README.md b/hack/aks/README.md index ce29df2927..a9877bb64f 100644 --- a/hack/aks/README.md +++ b/hack/aks/README.md @@ -21,16 +21,18 @@ SWIFT Infra net-up Create required swift vnet/subnets AKS Clusters - byocni-up Alias to swift-byocni-up - cilium-up Alias to swift-cilium-up - up Alias to swift-up - overlay-up Brings up an Overlay AzCNI cluster - swift-byocni-up Bring up a SWIFT BYO CNI cluster - swift-cilium-up Bring up a SWIFT Cilium cluster - swift-up Bring up a SWIFT AzCNI cluster - windows-cniv1-up Bring up a Windows AzCNIv1 cluster - dualstack-overlay-up Brings up an dualstack overlay cluster - dualstack-windows-up Brings up windows nodes on dualstack overlay cluster - down Delete the cluster - vmss-restart Restart the nodes of the cluster + byocni-up Alias to swift-byocni-up + cilium-up Alias to swift-cilium-up + up Alias to swift-up + overlay-up Brings up an Overlay AzCNI cluster + swift-byocni-up Bring up a SWIFT BYO CNI cluster + swift-cilium-up Bring up a SWIFT Cilium cluster + swift-up Bring up a SWIFT AzCNI cluster + windows-cniv1-up Bring up a Windows AzCNIv1 cluster + dualstack-overlay-up Brings up an dualstack overlay cluster + dualstack-overlay-byocni-up Brings up an dualstack overlay cluster without CNS and CNI installed + dualstack-windows-up Brings up windows nodes on dualstack overlay cluster + dualstack-windows--byocni-up Brings up windows nodes on dualstack overlay cluster without CNS and CNI installed + down Delete the cluster + vmss-restart Restart the nodes of the cluster ``` From 18bc3a541518ebb02826a871f2be5897abe20254 Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Tue, 18 Jul 2023 17:19:27 -0400 Subject: [PATCH 40/45] add install azure-vnet --- .../dualstack-overlay/dualstackoverlay-e2e-step-template.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml index e5fe2c1e7c..ffde793800 100644 --- a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml @@ -54,8 +54,8 @@ steps: - script: | echo "Start DualStack overlay E2E Tests" + sudo -E env "PATH=$PATH" make test-integration CNS_VERSION=$(make cns-version) CNI_DROPGZ_VERSION=$(make cni-dropgz-version) INSTALL_CNS=true INSTALL_AZURE_VNET=true TEST_DROPGZ=${{ parameters.testDropgz }} kubectl get po -owide -A - sudo -E env "PATH=$PATH" make test-integration CNS_VERSION=$(make cns-version) CNI_DROPGZ_VERSION=$(make cni-dropgz-version) INSTALL_CNS=true INSTALL_AZILIUM=true TEST_DROPGZ=${{ parameters.testDropgz }} retryCountOnTaskFailure: 3 name: "dualStackOverlayTest" displayName: "Run DualStack overlay E2E" From d942050c28677bfc6509a8f722fd40d05ab5c469 Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Wed, 19 Jul 2023 12:11:22 -0400 Subject: [PATCH 41/45] install kube --- .../dualstackoverlay-e2e-step-template.yaml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml index ffde793800..dfea39b44c 100644 --- a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml @@ -49,16 +49,11 @@ steps: kubectl cluster-info kubectl get node kubectl get po -owide -A - name: "installKubectl" - displayName: "Install kubectl on AKS dualstack overlay cluster" - - - script: | - echo "Start DualStack overlay E2E Tests" sudo -E env "PATH=$PATH" make test-integration CNS_VERSION=$(make cns-version) CNI_DROPGZ_VERSION=$(make cni-dropgz-version) INSTALL_CNS=true INSTALL_AZURE_VNET=true TEST_DROPGZ=${{ parameters.testDropgz }} kubectl get po -owide -A retryCountOnTaskFailure: 3 - name: "dualStackOverlayTest" - displayName: "Run DualStack overlay E2E" + name: "installKubectl" + displayName: "Install kubectl on AKS dualstack overlay cluster" - script: | cd test/integration/load From a32e69d2eb45a85d3a9b8d141207192d52eab922 Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Wed, 19 Jul 2023 14:49:49 -0400 Subject: [PATCH 42/45] disable cleaning up dualstack overlay cluster --- .../dualstackoverlay-e2e-step-template.yaml | 82 ++++++------- hack/aks/Makefile | 2 +- .../manifests/cns/windows-daemonset.yaml | 116 ++++++++++++++++++ 3 files changed, 158 insertions(+), 42 deletions(-) create mode 100644 test/integration/manifests/cns/windows-daemonset.yaml diff --git a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml index dfea39b44c..6213b775a0 100644 --- a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml @@ -70,46 +70,46 @@ steps: name: "DualStack_Overlay_Linux_tests" displayName: "DualStack Overlay Linux Tests" - - task: AzureCLI@2 - inputs: - azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) - scriptLocation: "inlineScript" - scriptType: "bash" - addSpnToEnvironment: true - inlineScript: | - make -C ./hack/aks dualstack-windows-up AZCLI=az VM_SIZE=$(VM_SIZE) NODE_COUNT=$(NODE_COUNT) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) - echo "Windows node are successfully added to Dualstack Overlay Cluster" - kubectl cluster-info - kubectl get node - kubectl get po -owide -A - name: "Add_Windows_Node" - displayName: "Add windows node on DualStackOverlay cluster" + # - task: AzureCLI@2 + # inputs: + # azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) + # scriptLocation: "inlineScript" + # scriptType: "bash" + # addSpnToEnvironment: true + # inlineScript: | + # make -C ./hack/aks dualstack-windows-byocni-up AZCLI=az VM_SIZE=$(VM_SIZE) NODE_COUNT=$(NODE_COUNT) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) + # echo "Windows node are successfully added to Dualstack Overlay Cluster" + # kubectl cluster-info + # kubectl get node + # kubectl get po -owide -A + # name: "Add_Windows_Node" + # displayName: "Add windows node on DualStackOverlay cluster" - - script: | - pwd - cd test/integration/load - go test -timeout 30m -tags load -run ^TestLoad$ -tags=load -os=windows - echo "DualStack Overlay Windows control plane CNS validation test" - go test -timeout 30m -tags load -run ^TestDualStackProperties$ -tags=load -os=windows -cni cniv2 - echo "DualStack Overlay Windows control plane Node properties test" - go test -timeout 30m -tags load -cni cniv2 -run ^TestValidateState$ -tags=load -os=windows - echo "DualStack Overlay Windows datapath test" - cd ../datapath - go test -count=1 datapath_windows_test.go -timeout 3m -tags connection -run ^TestDatapathWin$ -tags=connection - name: "DualStack_Overlay_Windows_tests" - displayName: "DualStack Overlay Windows Tests" + # - script: | + # pwd + # cd test/integration/load + # go test -timeout 30m -tags load -run ^TestLoad$ -tags=load -os=windows + # echo "DualStack Overlay Windows control plane CNS validation test" + # go test -timeout 30m -tags load -run ^TestDualStackProperties$ -tags=load -os=windows -cni cniv2 + # echo "DualStack Overlay Windows control plane Node properties test" + # go test -timeout 30m -tags load -cni cniv2 -run ^TestValidateState$ -tags=load -os=windows + # echo "DualStack Overlay Windows datapath test" + # cd ../datapath + # go test -count=1 datapath_windows_test.go -timeout 3m -tags connection -run ^TestDatapathWin$ -tags=connection + # name: "DualStack_Overlay_Windows_tests" + # displayName: "DualStack Overlay Windows Tests" - - task: AzureCLI@2 - inputs: - azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) - scriptLocation: "inlineScript" - scriptType: "bash" - addSpnToEnvironment: true - inlineScript: | - echo "Deleting cluster" - make -C ./hack/aks azcfg AZCLI=az - make -C ./hack/aks down SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision) - echo "Cluster and resources down" - name: "CleanupDualStackOverlaycluster" - displayName: "Cleanup DualStack Overlay Cluster" - condition: always() \ No newline at end of file + # - task: AzureCLI@2 + # inputs: + # azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) + # scriptLocation: "inlineScript" + # scriptType: "bash" + # addSpnToEnvironment: true + # inlineScript: | + # echo "Deleting cluster" + # make -C ./hack/aks azcfg AZCLI=az + # make -C ./hack/aks down SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision) + # echo "Cluster and resources down" + # name: "CleanupDualStackOverlaycluster" + # displayName: "Cleanup DualStack Overlay Cluster" + # condition: always() \ No newline at end of file diff --git a/hack/aks/Makefile b/hack/aks/Makefile index 968c87b6d1..c352d49edd 100644 --- a/hack/aks/Makefile +++ b/hack/aks/Makefile @@ -251,7 +251,7 @@ dualstack-windows-up: ## Brings up windows nodes on dualstack overlay cluster --subscription $(SUB) @$(MAKE) set-kubeconf -dualstack-windows--byocni-up: ## Brings up windows nodes on dualstack overlay cluster without CNS and CNI installed +dualstack-windows-byocni-up: ## Brings up windows nodes on dualstack overlay cluster without CNS and CNI installed $(AZCLI) aks nodepool add -g $(GROUP) -n npwin \ --cluster-name $(CLUSTER) \ --node-count $(NODE_COUNT) \ diff --git a/test/integration/manifests/cns/windows-daemonset.yaml b/test/integration/manifests/cns/windows-daemonset.yaml new file mode 100644 index 0000000000..cf9cf33c46 --- /dev/null +++ b/test/integration/manifests/cns/windows-daemonset.yaml @@ -0,0 +1,116 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: azure-cns + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: EnsureExists +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: azure-cns-win + namespace: kube-system + labels: + app: azure-cns-win +spec: + selector: + matchLabels: + k8s-app: azure-cns-win + template: + metadata: + labels: + k8s-app: azure-cns-win + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + prometheus.io/port: "10092" + spec: + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\SYSTEM" + containers: + - name: cns-container + image: acnpublic.azurecr.io/azure-cns:v1.4.26-9-gc40fb852 + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + command: ["powershell.exe"] + args: + [ + '.\setkubeconfigpath.ps1', ";", + 'powershell.exe', '.\azure-cns.exe', + '-c', "tcp://$(CNSIpAddress):$(CNSPort)", + '-t', "$(CNSLogTarget)", + '-o', "$(CNSLogDir)", + '-storefilepath', "$(CNSStoreFilePath)", + '-config-path', "%CONTAINER_SANDBOX_MOUNT_POINT%\\$(CNS_CONFIGURATION_PATH)", + '--kubeconfig', '.\kubeconfig', + ] + volumeMounts: + - name: log + mountPath: /k + - name: cns-config + mountPath: etc/azure-cns + ports: + - containerPort: 10090 + name: api + - containerPort: 10092 + name: metrics + env: + - name: CNSIpAddress + value: "127.0.0.1" + - name: CNSPort + value: "10090" + - name: CNSLogTarget + value: "stdoutfile" + - name: CNSLogDir + value: /k + - name: CNSStoreFilePath + value: /k/ + - name: CNS_CONFIGURATION_PATH + value: etc/azure-cns/cns_config.json + - name: NODENAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + hostNetwork: true + volumes: + - name: log + hostPath: + path: /k + type: Directory + - name: cns-config + configMap: + name: cns-config + nodeSelector: + kubernetes.io/os: windows + serviceAccountName: azure-cns +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: cns-config + namespace: kube-system +data: + cns_config.json: | + { + "TelemetrySettings": { + "TelemetryBatchSizeBytes": 16384, + "TelemetryBatchIntervalInSecs": 15, + "RefreshIntervalInSecs": 15, + "DisableAll": false, + "HeartBeatIntervalInMins": 30, + "DebugMode": false, + "SnapshotIntervalInMins": 60 + }, + "ManagedSettings": { + "PrivateEndpoint": "", + "InfrastructureNetworkID": "", + "NodeID": "", + "NodeSyncIntervalInSeconds": 30 + }, + "ChannelMode": "CRD", + "InitializeFromCNI": true + } \ No newline at end of file From 5b8ae6dea29f2df1413318b36406a0d1268a0d84 Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Thu, 20 Jul 2023 11:25:46 -0400 Subject: [PATCH 43/45] fix dualstack overlay linux dropgz conflist file --- .../dualstackoverlay-e2e-step-template.yaml | 2 +- test/integration/setup_test.go | 28 ++++++++++++++----- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml index 6213b775a0..73508334b7 100644 --- a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml @@ -49,7 +49,7 @@ steps: kubectl cluster-info kubectl get node kubectl get po -owide -A - sudo -E env "PATH=$PATH" make test-integration CNS_VERSION=$(make cns-version) CNI_DROPGZ_VERSION=$(make cni-dropgz-version) INSTALL_CNS=true INSTALL_AZURE_VNET=true TEST_DROPGZ=${{ parameters.testDropgz }} + sudo -E env "PATH=$PATH" make test-integration CNS_VERSION=$(make cns-version) CNI_DROPGZ_VERSION=$(make cni-dropgz-version) INSTALL_CNS=true INSTALL_AZURE_VNET=true INSTALL_DUALSTACK_OVERLAY=true TEST_DROPGZ=${{ parameters.testDropgz }} kubectl get po -owide -A retryCountOnTaskFailure: 3 name: "installKubectl" diff --git a/test/integration/setup_test.go b/test/integration/setup_test.go index 5ece6e6498..ff67f7c784 100644 --- a/test/integration/setup_test.go +++ b/test/integration/setup_test.go @@ -17,13 +17,14 @@ import ( const ( exitFail = 1 - envTestDropgz = "TEST_DROPGZ" - envCNIDropgzVersion = "CNI_DROPGZ_VERSION" - envCNSVersion = "CNS_VERSION" - envInstallCNS = "INSTALL_CNS" - envInstallAzilium = "INSTALL_AZILIUM" - envInstallAzureVnet = "INSTALL_AZURE_VNET" - envInstallOverlay = "INSTALL_OVERLAY" + envTestDropgz = "TEST_DROPGZ" + envCNIDropgzVersion = "CNI_DROPGZ_VERSION" + envCNSVersion = "CNS_VERSION" + envInstallCNS = "INSTALL_CNS" + envInstallAzilium = "INSTALL_AZILIUM" + envInstallAzureVnet = "INSTALL_AZURE_VNET" + envInstallOverlay = "INSTALL_OVERLAY" + envInstallDualStackOverlay = "INSTALL_DUALSTACK_OVERLAY" // relative cns manifest paths cnsManifestFolder = "manifests/cns" @@ -160,6 +161,19 @@ func installCNSDaemonset(ctx context.Context, clientset *kubernetes.Clientset, l log.Printf("Env %v not set to true, skipping", envInstallOverlay) } + if installBool4 := os.Getenv(envInstallDualStackOverlay); installBool4 != "" { + if dualStackOverlayScenario, err := strconv.ParseBool(installBool4); err == nil && dualStackOverlayScenario == true { + log.Printf("Env %v set to true, deploy azure-vnet", envInstallDualStackOverlay) + cns.Spec.Template.Spec.InitContainers[0].Args = []string{"deploy", "azure-vnet", "-o", "/opt/cni/bin/azure-vnet", "azure-vnet-telemetry", "-o", "/opt/cni/bin/azure-vnet-telemetry", "azure-vnet-ipam", "-o", "/opt/cni/bin/azure-vnet-ipam", "azure-swift-overlay-dualstack.conflist", "-o", "/etc/cni/net.d/10-azure.conflist"} + } + // setup the CNS swiftconfigmap + if err := k8sutils.MustSetupConfigMap(ctx, clientset, cnsSwiftConfigMapPath); err != nil { + return nil, err + } + } else { + log.Printf("Env %v not set to true, skipping", envInstallDualStackOverlay) + } + cnsDaemonsetClient := clientset.AppsV1().DaemonSets(cns.Namespace) log.Printf("Installing CNS with image %s", cns.Spec.Template.Spec.Containers[0].Image) From 892012c829a002e06480363150e11ccf820130dd Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Thu, 20 Jul 2023 15:35:42 -0400 Subject: [PATCH 44/45] fix dropgz image --- .../dualstackoverlay-e2e-step-template.yaml | 2 +- Makefile | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml index 73508334b7..49770d80f1 100644 --- a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml @@ -49,7 +49,7 @@ steps: kubectl cluster-info kubectl get node kubectl get po -owide -A - sudo -E env "PATH=$PATH" make test-integration CNS_VERSION=$(make cns-version) CNI_DROPGZ_VERSION=$(make cni-dropgz-version) INSTALL_CNS=true INSTALL_AZURE_VNET=true INSTALL_DUALSTACK_OVERLAY=true TEST_DROPGZ=${{ parameters.testDropgz }} + sudo -E env "PATH=$PATH" make install-azure-images CNS_VERSION=$(make cns-version) CNI_DROPGZ_VERSION=$(make cni-dropgz-version) INSTALL_CNS=true INSTALL_AZURE_VNET=true INSTALL_DUALSTACK_OVERLAY=true TEST_DROPGZ=${{ parameters.testDropgz }} kubectl get po -owide -A retryCountOnTaskFailure: 3 name: "installKubectl" diff --git a/Makefile b/Makefile index f8b155324d..eca81a1558 100644 --- a/Makefile +++ b/Makefile @@ -762,6 +762,11 @@ test-azure-ipam: ## run the unit test for azure-ipam kind: kind create cluster --config ./test/kind/kind.yaml +# install azure Linux CNS and CNI dropgz images +install-azure-images: + CNI_DROPGZ_VERSION=$(CNI_DROPGZ_VERSION) \ + CNS_VERSION=$(CNS_VERSION) \ + go test -mod=readonly -buildvcs=false -timeout 1h -coverpkg=./... -race -covermode atomic -coverprofile=coverage.out -tags=integration ./test/integration/setup_test.go ##@ Utilities From 0fb5594d0db949abadee6a7302fe5cce4b410433 Mon Sep 17 00:00:00 2001 From: paulyufan2 Date: Thu, 20 Jul 2023 17:31:08 -0400 Subject: [PATCH 45/45] add sudo to run test cases --- .../dualstackoverlay-e2e-step-template.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml index 49770d80f1..d9371695c7 100644 --- a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml @@ -57,14 +57,14 @@ steps: - script: | cd test/integration/load - go test -timeout 30m -tags load -run ^TestLoad$ -tags=load + sudo go test -timeout 30m -tags load -run ^TestLoad$ -tags=load echo "DualStack Overlay Linux control plane CNS validation test" - go test -timeout 30m -tags load -cni cniv2 -run ^TestValidateState$ -tags=load + sudo go test -timeout 30m -tags load -cni cniv2 -run ^TestValidateState$ -tags=load echo "DualStack Overlay Linux control plane Node properties test" - go test -timeout 30m -tags load -run ^TestDualStackProperties$ -tags=load + sudo go test -timeout 30m -tags load -run ^TestDualStackProperties$ -tags=load echo "DualStack Overlay Linux datapath test" cd ../datapath - go test -count=1 datapath_linux_test.go -timeout 1m -tags connection -run ^TestDatapathLinux$ -tags=connection,integration + sudo go test -count=1 datapath_linux_test.go -timeout 1m -tags connection -run ^TestDatapathLinux$ -tags=connection,integration echo "Delete Linux load-test namespace" kubectl delete ns load-test name: "DualStack_Overlay_Linux_tests"