diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml index e77a807344..db711641ec 100644 --- a/.pipelines/pipeline.yaml +++ b/.pipelines/pipeline.yaml @@ -332,6 +332,14 @@ stages: testDropgz: "" clusterName: "overlaye2e" + - template: singletenancy/dualstack-overlay/dualstackoverlay-e2e-job-template.yaml + parameters: + name: "dualstackoverlay_e2e" + displayName: AKS DualStack Overlay + pipelineBuildImage: "$(BUILD_IMAGE)" + testDropgz: "" + clusterName: "dualstackoverlaye2e" + - template: singletenancy/aks-swift/e2e-job-template.yaml parameters: name: "aks_swift_e2e" diff --git a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-job-template.yaml b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-job-template.yaml new file mode 100644 index 0000000000..fadb3196ab --- /dev/null +++ b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-job-template.yaml @@ -0,0 +1,32 @@ +parameters: + name: "" + displayName: "" + pipelineBuildImage: "$(BUILD_IMAGE)" + testDropgz: "" + clusterName: "" + +stages: + - stage: ${{ parameters.name }} + displayName: E2E - ${{ parameters.displayName }} + dependsOn: + - setup + - publish + jobs: + - job: ${{ parameters.name }} + displayName: DualStack Overlay Test Suite - (${{ parameters.name }}) + timeoutInMinutes: 120 + pool: + name: $(BUILD_POOL_NAME_DEFAULT) + demands: + - Role -equals $(CUSTOM_E2E_ROLE) + variables: + GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path + GOBIN: "$(GOPATH)/bin" # Go binaries path + modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking" + steps: + - template: dualstackoverlay-e2e-step-template.yaml + parameters: + name: ${{ parameters.name }} + testDropgz: ${{ parameters.testDropgz }} + clusterName: ${{ parameters.clusterName }} + \ No newline at end of file diff --git a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml new file mode 100644 index 0000000000..d9371695c7 --- /dev/null +++ b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml @@ -0,0 +1,115 @@ +parameters: + name: "" + testDropgz: "" + clusterName: "" + +steps: + - bash: | + echo $UID + sudo rm -rf $(System.DefaultWorkingDirectory)/* + displayName: "Set up OS environment" + - checkout: self + + - bash: | + go version + go env + mkdir -p '$(GOBIN)' + mkdir -p '$(GOPATH)/pkg' + mkdir -p '$(modulePath)' + echo '##vso[task.prependpath]$(GOBIN)' + echo '##vso[task.prependpath]$(GOROOT)/bin' + name: "GoEnv" + displayName: "Set up the Go environment" + - task: AzureCLI@2 + inputs: + azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + echo "Check az version" + az version + echo "Install az cli extension preview" + az extension add --name aks-preview + az extension update --name aks-preview + mkdir -p ~/.kube/ + echo "Create AKS DualStack Overlay cluster" + make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_OVERLAY_CLUSTER_TEST) + make -C ./hack/aks dualstack-overlay-byocni-up AZCLI=az REGION=$(REGION_OVERLAY_CLUSTER_TEST) VM_SIZE=$(VM_SIZE) NODE_COUNT=$(NODE_COUNT) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) + echo "Dualstack Overlay Cluster is successfully created" + displayName: Create DualStackOverlay cluster + condition: succeeded() + + - script: | + ls -lah + pwd + echo "installing kubectl" + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl + kubectl cluster-info + kubectl get node + kubectl get po -owide -A + sudo -E env "PATH=$PATH" make install-azure-images CNS_VERSION=$(make cns-version) CNI_DROPGZ_VERSION=$(make cni-dropgz-version) INSTALL_CNS=true INSTALL_AZURE_VNET=true INSTALL_DUALSTACK_OVERLAY=true TEST_DROPGZ=${{ parameters.testDropgz }} + kubectl get po -owide -A + retryCountOnTaskFailure: 3 + name: "installKubectl" + displayName: "Install kubectl on AKS dualstack overlay cluster" + + - script: | + cd test/integration/load + sudo go test -timeout 30m -tags load -run ^TestLoad$ -tags=load + echo "DualStack Overlay Linux control plane CNS validation test" + sudo go test -timeout 30m -tags load -cni cniv2 -run ^TestValidateState$ -tags=load + echo "DualStack Overlay Linux control plane Node properties test" + sudo go test -timeout 30m -tags load -run ^TestDualStackProperties$ -tags=load + echo "DualStack Overlay Linux datapath test" + cd ../datapath + sudo go test -count=1 datapath_linux_test.go -timeout 1m -tags connection -run ^TestDatapathLinux$ -tags=connection,integration + echo "Delete Linux load-test namespace" + kubectl delete ns load-test + name: "DualStack_Overlay_Linux_tests" + displayName: "DualStack Overlay Linux Tests" + + # - task: AzureCLI@2 + # inputs: + # azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) + # scriptLocation: "inlineScript" + # scriptType: "bash" + # addSpnToEnvironment: true + # inlineScript: | + # make -C ./hack/aks dualstack-windows-byocni-up AZCLI=az VM_SIZE=$(VM_SIZE) NODE_COUNT=$(NODE_COUNT) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) + # echo "Windows node are successfully added to Dualstack Overlay Cluster" + # kubectl cluster-info + # kubectl get node + # kubectl get po -owide -A + # name: "Add_Windows_Node" + # displayName: "Add windows node on DualStackOverlay cluster" + + # - script: | + # pwd + # cd test/integration/load + # go test -timeout 30m -tags load -run ^TestLoad$ -tags=load -os=windows + # echo "DualStack Overlay Windows control plane CNS validation test" + # go test -timeout 30m -tags load -run ^TestDualStackProperties$ -tags=load -os=windows -cni cniv2 + # echo "DualStack Overlay Windows control plane Node properties test" + # go test -timeout 30m -tags load -cni cniv2 -run ^TestValidateState$ -tags=load -os=windows + # echo "DualStack Overlay Windows datapath test" + # cd ../datapath + # go test -count=1 datapath_windows_test.go -timeout 3m -tags connection -run ^TestDatapathWin$ -tags=connection + # name: "DualStack_Overlay_Windows_tests" + # displayName: "DualStack Overlay Windows Tests" + + # - task: AzureCLI@2 + # inputs: + # azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION) + # scriptLocation: "inlineScript" + # scriptType: "bash" + # addSpnToEnvironment: true + # inlineScript: | + # echo "Deleting cluster" + # make -C ./hack/aks azcfg AZCLI=az + # make -C ./hack/aks down SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision) + # echo "Cluster and resources down" + # name: "CleanupDualStackOverlaycluster" + # displayName: "Cleanup DualStack Overlay Cluster" + # condition: always() \ No newline at end of file diff --git a/Makefile b/Makefile index f8b155324d..eca81a1558 100644 --- a/Makefile +++ b/Makefile @@ -762,6 +762,11 @@ test-azure-ipam: ## run the unit test for azure-ipam kind: kind create cluster --config ./test/kind/kind.yaml +# install azure Linux CNS and CNI dropgz images +install-azure-images: + CNI_DROPGZ_VERSION=$(CNI_DROPGZ_VERSION) \ + CNS_VERSION=$(CNS_VERSION) \ + go test -mod=readonly -buildvcs=false -timeout 1h -coverpkg=./... -race -covermode atomic -coverprofile=coverage.out -tags=integration ./test/integration/setup_test.go ##@ Utilities diff --git a/hack/aks/Makefile b/hack/aks/Makefile index 73233657af..c352d49edd 100644 --- a/hack/aks/Makefile +++ b/hack/aks/Makefile @@ -214,6 +214,54 @@ windows-cniv1-up: rg-up overlay-net-up ## Bring up a Windows CNIv1 cluster @$(MAKE) set-kubeconf +dualstack-overlay-up: rg-up overlay-net-up ## Brings up an dualstack Overlay cluster with Linux node only + $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ + --kubernetes-version 1.26.3 \ + --node-count $(NODE_COUNT) \ + --node-vm-size $(VM_SIZE) \ + --network-plugin azure \ + --ip-families ipv4,ipv6 \ + --network-plugin-mode overlay \ + --subscription $(SUB) \ + --no-ssh-key \ + --yes + @$(MAKE) set-kubeconf + +dualstack-overlay-byocni-up: rg-up overlay-net-up ## Brings up an dualstack Overlay BYO CNI cluster + $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ + --kubernetes-version 1.26.3 \ + --node-count $(NODE_COUNT) \ + --node-vm-size $(VM_SIZE) \ + --network-plugin none \ + --network-plugin-mode overlay \ + --aks-custom-headers AKSHTTPCustomFeatures=Microsoft.ContainerService/AzureOverlayDualStackPreview \ + --ip-families ipv4,ipv6 \ + --subscription $(SUB) \ + --no-ssh-key \ + --yes + @$(MAKE) set-kubeconf + +dualstack-windows-up: ## Brings up windows nodes on dualstack overlay cluster + $(AZCLI) aks nodepool add -g $(GROUP) -n npwin \ + --cluster-name $(CLUSTER) \ + --node-count $(NODE_COUNT) \ + --node-vm-size $(VM_SIZE) \ + --os-type Windows \ + --os-sku Windows2022 \ + --subscription $(SUB) + @$(MAKE) set-kubeconf + +dualstack-windows-byocni-up: ## Brings up windows nodes on dualstack overlay cluster without CNS and CNI installed + $(AZCLI) aks nodepool add -g $(GROUP) -n npwin \ + --cluster-name $(CLUSTER) \ + --node-count $(NODE_COUNT) \ + --node-vm-size $(VM_SIZE) \ + --network-plugin none \ + --os-type Windows \ + --os-sku Windows2022 \ + --subscription $(SUB) + @$(MAKE) set-kubeconf + linux-cniv1-up: rg-up overlay-net-up $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ --node-count $(NODE_COUNT) \ diff --git a/hack/aks/README.md b/hack/aks/README.md index 3a4e80e4f0..a9877bb64f 100644 --- a/hack/aks/README.md +++ b/hack/aks/README.md @@ -21,14 +21,18 @@ SWIFT Infra net-up Create required swift vnet/subnets AKS Clusters - byocni-up Alias to swift-byocni-up - cilium-up Alias to swift-cilium-up - up Alias to swift-up - overlay-up Brings up an Overlay AzCNI cluster - swift-byocni-up Bring up a SWIFT BYO CNI cluster - swift-cilium-up Bring up a SWIFT Cilium cluster - swift-up Bring up a SWIFT AzCNI cluster - windows-cniv1-up Bring up a Windows AzCNIv1 cluster - down Delete the cluster - vmss-restart Restart the nodes of the cluster + byocni-up Alias to swift-byocni-up + cilium-up Alias to swift-cilium-up + up Alias to swift-up + overlay-up Brings up an Overlay AzCNI cluster + swift-byocni-up Bring up a SWIFT BYO CNI cluster + swift-cilium-up Bring up a SWIFT Cilium cluster + swift-up Bring up a SWIFT AzCNI cluster + windows-cniv1-up Bring up a Windows AzCNIv1 cluster + dualstack-overlay-up Brings up an dualstack overlay cluster + dualstack-overlay-byocni-up Brings up an dualstack overlay cluster without CNS and CNI installed + dualstack-windows-up Brings up windows nodes on dualstack overlay cluster + dualstack-windows--byocni-up Brings up windows nodes on dualstack overlay cluster without CNS and CNI installed + down Delete the cluster + vmss-restart Restart the nodes of the cluster ``` diff --git a/test/integration/datapath/datapath_linux_test.go b/test/integration/datapath/datapath_linux_test.go new file mode 100644 index 0000000000..a0a4ab9a38 --- /dev/null +++ b/test/integration/datapath/datapath_linux_test.go @@ -0,0 +1,325 @@ +//go:build connection + +package connection + +import ( + "context" + "flag" + "fmt" + "net" + "os" + "testing" + "time" + + "github.com/Azure/azure-container-networking/test/integration" + "github.com/Azure/azure-container-networking/test/integration/goldpinger" + k8sutils "github.com/Azure/azure-container-networking/test/internal/k8sutils" + "github.com/Azure/azure-container-networking/test/internal/retry" + "github.com/pkg/errors" + "github.com/stretchr/testify/require" + + appsv1 "k8s.io/api/apps/v1" + apiv1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + LinuxDeployIPV4 = "../manifests/datapath/linux-deployment.yaml" + LinuxDeployIPv6 = "../manifests/datapath/linux-deployment-ipv6.yaml" + podLabelKey = "app" + podCount = 2 + nodepoolKey = "agentpool" + maxRetryDelaySeconds = 10 + defaultTimeoutSeconds = 120 + defaultRetryDelaySeconds = 1 + goldpingerRetryCount = 24 + goldpingerDelayTimeSeconds = 5 + gpFolder = "../manifests/goldpinger" + gpClusterRolePath = gpFolder + "/cluster-role.yaml" + gpClusterRoleBindingPath = gpFolder + "/cluster-role-binding.yaml" + gpServiceAccountPath = gpFolder + "/service-account.yaml" + gpDaemonset = gpFolder + "/daemonset.yaml" + gpDaemonsetIPv6 = gpFolder + "/daemonset-ipv6.yaml" + gpDeployment = gpFolder + "/deployment.yaml" +) + +var ( + podPrefix = flag.String("podName", "goldpinger", "Prefix for test pods") + podNamespace = flag.String("namespace", "default", "Namespace for test pods") + nodepoolSelector = flag.String("nodepoolSelector", "nodepool1", "Provides nodepool as a Node-Selector for pods") + testProfile = flag.String("testName", LinuxDeployIPV4, "Linux datapath test profile") + defaultRetrier = retry.Retrier{ + Attempts: 10, + Delay: defaultRetryDelaySeconds * time.Second, + } +) + +/* +This test assumes that you have the current credentials loaded in your default kubeconfig for a +k8s cluster with a Linux nodepool consisting of at least 2 Linux nodes. +*** The expected nodepool name is npwin, if the nodepool has a diferent name ensure that you change nodepoolSelector with: + -nodepoolSelector="yournodepoolname" + +To run the test use one of the following commands: +go test -count=1 test/integration/datapath/datapath_linux_test.go -timeout 3m -tags connection -run ^TestDatapathLinux$ -tags=connection,integration + or +go test -count=1 test/integration/datapath/datapath_linux_test.go -timeout 3m -tags connection -run ^TestDatapathLinux$ -podName=acnpod -nodepoolSelector=aks-pool1 -tags=connection,integration + + +This test checks pod to pod, pod to node, pod to Internet check + +Timeout context is controled by the -timeout flag. + +*/ + +// return podLabelSelector and nodeLabelSelector +func createLabelSelectors() (string, string) { + return fmt.Sprintf("%s=%s", podLabelKey, *podPrefix), fmt.Sprintf("%s=%s", nodepoolKey, *nodepoolSelector) +} + +func setupLinuxEnvironment(t *testing.T) { + ctx := context.Background() + + t.Log("Create Clientset") + clientset, err := k8sutils.MustGetClientset() + if err != nil { + require.NoError(t, err, "could not get k8s clientset: %v", err) + } + + t.Log("Create Label Selectors") + podLabelSelector, nodeLabelSelector := createLabelSelectors() + + t.Log("Get Nodes") + nodes, err := k8sutils.GetNodeListByLabelSelector(ctx, clientset, nodeLabelSelector) + if err != nil { + require.NoError(t, err, "could not get k8s node list: %v", err) + } + + createPodFlag := !(apierrors.IsAlreadyExists(err)) + t.Logf("%v", createPodFlag) + + if createPodFlag { + var daemonset appsv1.DaemonSet + t.Log("Creating Linux pods through deployment") + deployment, err := k8sutils.MustParseDeployment(*testProfile) + if err != nil { + require.NoError(t, err) + } + + if *testProfile == LinuxDeployIPV4 { + daemonset, err = k8sutils.MustParseDaemonSet(gpDaemonset) + if err != nil { + t.Fatal(err) + } + } else { + daemonset, err = k8sutils.MustParseDaemonSet(gpDaemonsetIPv6) + if err != nil { + t.Fatal(err) + } + } + + rbacCleanUpFn, err := k8sutils.MustSetUpClusterRBAC(ctx, clientset, gpClusterRolePath, gpClusterRoleBindingPath, gpServiceAccountPath) + if err != nil { + t.Log(os.Getwd()) + t.Fatal(err) + } + + // Fields for overwritting existing deployment yaml. + // Defaults from flags will not change anything + deployment.Spec.Selector.MatchLabels[podLabelKey] = *podPrefix + deployment.Spec.Template.ObjectMeta.Labels[podLabelKey] = *podPrefix + deployment.Spec.Template.Spec.NodeSelector[nodepoolKey] = *nodepoolSelector + deployment.Name = *podPrefix + deployment.Namespace = *podNamespace + + deploymentsClient := clientset.AppsV1().Deployments(*podNamespace) + err = k8sutils.MustCreateDeployment(ctx, deploymentsClient, deployment) + if err != nil { + require.NoError(t, err) + } + + daemonsetClient := clientset.AppsV1().DaemonSets(daemonset.Namespace) + err = k8sutils.MustCreateDaemonset(ctx, daemonsetClient, daemonset) + if err != nil { + t.Fatal(err) + } + + t.Cleanup(func() { + t.Log("cleaning up resources") + rbacCleanUpFn() + + if err := deploymentsClient.Delete(ctx, deployment.Name, metav1.DeleteOptions{}); err != nil { + t.Log(err) + } + + if err := daemonsetClient.Delete(ctx, daemonset.Name, metav1.DeleteOptions{}); err != nil { + t.Log(err) + } + }) + + t.Log("Waiting for pods to be running state") + err = k8sutils.WaitForPodsRunning(ctx, clientset, *podNamespace, podLabelSelector) + if err != nil { + require.NoError(t, err) + } + t.Log("Successfully created customer linux pods") + } else { + t.Log("Checking for pods to be running state") + err = k8sutils.WaitForPodsRunning(ctx, clientset, *podNamespace, podLabelSelector) + if err != nil { + require.NoError(t, err) + } + } + + t.Log("Checking Linux test environment") + for _, node := range nodes.Items { + pods, err := k8sutils.GetPodsByNode(ctx, clientset, *podNamespace, podLabelSelector, node.Name) + if err != nil { + require.NoError(t, err, "could not get k8s clientset: %v", err) + } + if len(pods.Items) <= 1 { + t.Logf("%s", node.Name) + require.NoError(t, errors.New("Less than 2 pods on node")) + } + + } + t.Log("Linux test environment ready") +} + +func TestDatapathLinux(t *testing.T) { + ctx := context.Background() + + t.Log("Get REST config") + restConfig := k8sutils.MustGetRestConfig(t) + + t.Log("Create Clientset") + clientset, _ := k8sutils.MustGetClientset() + + setupLinuxEnvironment(t) + podLabelSelector, _ := createLabelSelectors() + + t.Run("Linux ping tests", func(t *testing.T) { + // Check goldpinger health + t.Run("all pods have IPs assigned", func(t *testing.T) { + podsClient := clientset.CoreV1().Pods(*podNamespace) + + checkPodIPsFn := func() error { + podList, err := podsClient.List(ctx, metav1.ListOptions{LabelSelector: "app=goldpinger"}) + if err != nil { + return err + } + + if len(podList.Items) == 0 { + return errors.New("no pods scheduled") + } + + for _, pod := range podList.Items { + if pod.Status.Phase == apiv1.PodPending { + return errors.New("some pods still pending") + } + } + + for _, pod := range podList.Items { + if pod.Status.PodIP == "" { + return errors.New("a pod has not been allocated an IP") + } + } + + return nil + } + err := defaultRetrier.Do(ctx, checkPodIPsFn) + if err != nil { + t.Fatalf("not all pods were allocated IPs: %v", err) + } + t.Log("all pods have been allocated IPs") + }) + + // TODO: avoid using yaml file path to control test case + if *testProfile == LinuxDeployIPv6 { + t.Run("Linux dualstack overlay tests", func(t *testing.T) { + t.Run("test dualstack overlay", func(t *testing.T) { + podsClient := clientset.CoreV1().Pods(*podNamespace) + + checkPodIPsFn := func() error { + podList, err := podsClient.List(ctx, metav1.ListOptions{LabelSelector: "app=goldpinger"}) + if err != nil { + return err + } + + for _, pod := range podList.Items { + podIPs := pod.Status.PodIPs + if len(podIPs) < 2 { + return errors.New("a pod only gets one IP") + } + if net.ParseIP(podIPs[0].IP).To4() == nil || net.ParseIP(podIPs[1].IP).To16() == nil { + return errors.New("a pod does not have both ipv4 and ipv6 address") + } + } + return nil + } + err := defaultRetrier.Do(ctx, checkPodIPsFn) + if err != nil { + t.Fatalf("dualstack overlay pod properties check is failed due to: %v", err) + } + + t.Log("all dualstack linux pods properties have been verified") + }) + }) + } + + t.Run("all linux pods can ping each other", func(t *testing.T) { + clusterCheckCtx, cancel := context.WithTimeout(ctx, 3*time.Minute) + defer cancel() + + pfOpts := k8s.PortForwardingOpts{ + Namespace: *podNamespace, + LabelSelector: podLabelSelector, + LocalPort: 9090, + DestPort: 8080, + } + + pf, err := k8s.NewPortForwarder(restConfig, t, pfOpts) + if err != nil { + t.Fatal(err) + } + + portForwardCtx, cancel := context.WithTimeout(ctx, defaultTimeoutSeconds*time.Second) + defer cancel() + + portForwardFn := func() error { + err := pf.Forward(portForwardCtx) + if err != nil { + t.Logf("unable to start port forward: %v", err) + return err + } + return nil + } + if err := defaultRetrier.Do(portForwardCtx, portForwardFn); err != nil { + t.Fatalf("could not start port forward within %ds: %v", defaultTimeoutSeconds, err) + } + defer pf.Stop() + + gpClient := goldpinger.Client{Host: pf.Address()} + clusterCheckFn := func() error { + clusterState, err := gpClient.CheckAll(clusterCheckCtx) + if err != nil { + return err + } + stats := goldpinger.ClusterStats(clusterState) + stats.PrintStats() + if stats.AllPingsHealthy() { + return nil + } + + return errors.New("not all pings are healthy") + } + retrier := retry.Retrier{Attempts: goldpingerRetryCount, Delay: goldpingerDelayTimeSeconds * time.Second} + if err := retrier.Do(clusterCheckCtx, clusterCheckFn); err != nil { + t.Fatalf("goldpinger pods network health could not reach healthy state after %d seconds: %v", goldpingerRetryCount*goldpingerDelayTimeSeconds, err) + } + + t.Log("all pings successful!") + }) + }) +} diff --git a/test/integration/datapath/datapath_win_test.go b/test/integration/datapath/datapath_windows_test.go similarity index 88% rename from test/integration/datapath/datapath_win_test.go rename to test/integration/datapath/datapath_windows_test.go index 054a60bb98..f6f42713b0 100644 --- a/test/integration/datapath/datapath_win_test.go +++ b/test/integration/datapath/datapath_windows_test.go @@ -6,6 +6,7 @@ import ( "context" "flag" "fmt" + "net" "testing" "github.com/Azure/azure-container-networking/test/internal/datapath" @@ -36,9 +37,9 @@ k8s cluster with a windows nodepool consisting of at least 2 windows nodes. -nodepoolSelector="yournodepoolname" To run the test use one of the following commands: -go test -count=1 test/integration/datapath/datapath_win_test.go -timeout 3m -tags connection -run ^TestDatapathWin$ -tags=connection +go test -count=1 test/integration/datapath/datapath_windows_test.go -timeout 3m -tags connection -run ^TestDatapathWin$ -tags=connection or -go test -count=1 test/integration/datapath/datapath_win_test.go -timeout 3m -tags connection -run ^TestDatapathWin$ -podName=acnpod -nodepoolSelector=npwina -tags=connection +go test -count=1 test/integration/datapath/datapath_windows_test.go -timeout 3m -tags connection -run ^TestDatapathWin$ -podName=acnpod -nodepoolSelector=npwina -tags=connection This test checks pod to pod, pod to node, and pod to internet for datapath connectivity. @@ -110,7 +111,7 @@ func TestDatapathWin(t *testing.T) { require.NoError(t, err) } } - t.Log("Checking Windows test environment ") + t.Log("Checking Windows test environment") for _, node := range nodes.Items { pods, err := k8sutils.GetPodsByNode(ctx, clientset, *podNamespace, podLabelSelector, node.Name) @@ -129,18 +130,26 @@ func TestDatapathWin(t *testing.T) { for _, node := range nodes.Items { t.Log("Windows ping tests (1)") nodeIP := "" + nodeIPv6 := "" for _, address := range node.Status.Addresses { if address.Type == "InternalIP" { nodeIP = address.Address - // Multiple addresses exist, break once Internal IP found. - // Cannot call directly - break + if net.ParseIP(address.Address).To16() != nil { + nodeIPv6 = address.Address + } } } err := datapath.WindowsPodToNode(ctx, clientset, node.Name, nodeIP, *podNamespace, podLabelSelector, restConfig) require.NoError(t, err, "Windows pod to node, ping test failed with: %+v", err) t.Logf("Windows pod to node, passed for node: %s", node.Name) + + // windows ipv6 connectivity + if nodeIPv6 != "" { + err = datapath.WindowsPodToNode(ctx, clientset, node.Name, nodeIPv6, *podNamespace, podLabelSelector, restConfig) + require.NoError(t, err, "Windows pod to node, ipv6 ping test failed with: %+v", err) + t.Logf("Windows pod to node via ipv6, passed for node: %s", node.Name) + } } }) diff --git a/test/integration/goldpinger/client.go b/test/integration/goldpinger/client.go index 49b29d9686..dac4149ced 100644 --- a/test/integration/goldpinger/client.go +++ b/test/integration/goldpinger/client.go @@ -1,3 +1,4 @@ +//go:build integration // +build integration package goldpinger diff --git a/test/integration/load/load_test.go b/test/integration/load/load_test.go index ae09235eac..c734897256 100644 --- a/test/integration/load/load_test.go +++ b/test/integration/load/load_test.go @@ -28,6 +28,7 @@ var ( skipWait = flag.Bool("skip-wait", false, "Skip waiting for pods to be ready") restartCase = flag.Bool("restart-case", false, "In restart case, skip if we don't find state file") namespace = "load-test" + validateDualStack = flag.Bool("validate-dualstack", false, "Validate the dualstack overlay") ) var noopDeploymentMap = map[string]string{ @@ -116,6 +117,10 @@ func TestLoad(t *testing.T) { t.Fatal(err) } + if *validateDualStack { + t.Run("Validate dualstack overlay", TestDualStackProperties) + } + if *validateStateFile { t.Run("Validate state file", TestValidateState) } @@ -180,3 +185,23 @@ func TestScaleDeployment(t *testing.T) { t.Fatal(err) } } + +func TestDualStackProperties(t *testing.T) { + clientset, err := k8sutils.MustGetClientset() + if err != nil { + t.Fatal(err) + } + config := k8sutils.MustGetRestConfig(t) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) + defer cancel() + + t.Log("Validating the dualstack node labels") + validatorClient := validate.GetValidatorClient(*osType) + validator := validatorClient.CreateClient(ctx, clientset, config, namespace, *cniType, *restartCase) + + // validate dualstack overlay scenarios + err = validator.ValidateDualStackNodeProperties() + if err != nil { + t.Fatal(err) + } +} diff --git a/test/integration/manifests/cns/windows-daemonset.yaml b/test/integration/manifests/cns/windows-daemonset.yaml new file mode 100644 index 0000000000..cf9cf33c46 --- /dev/null +++ b/test/integration/manifests/cns/windows-daemonset.yaml @@ -0,0 +1,116 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: azure-cns + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: EnsureExists +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: azure-cns-win + namespace: kube-system + labels: + app: azure-cns-win +spec: + selector: + matchLabels: + k8s-app: azure-cns-win + template: + metadata: + labels: + k8s-app: azure-cns-win + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + prometheus.io/port: "10092" + spec: + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\SYSTEM" + containers: + - name: cns-container + image: acnpublic.azurecr.io/azure-cns:v1.4.26-9-gc40fb852 + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + command: ["powershell.exe"] + args: + [ + '.\setkubeconfigpath.ps1', ";", + 'powershell.exe', '.\azure-cns.exe', + '-c', "tcp://$(CNSIpAddress):$(CNSPort)", + '-t', "$(CNSLogTarget)", + '-o', "$(CNSLogDir)", + '-storefilepath', "$(CNSStoreFilePath)", + '-config-path', "%CONTAINER_SANDBOX_MOUNT_POINT%\\$(CNS_CONFIGURATION_PATH)", + '--kubeconfig', '.\kubeconfig', + ] + volumeMounts: + - name: log + mountPath: /k + - name: cns-config + mountPath: etc/azure-cns + ports: + - containerPort: 10090 + name: api + - containerPort: 10092 + name: metrics + env: + - name: CNSIpAddress + value: "127.0.0.1" + - name: CNSPort + value: "10090" + - name: CNSLogTarget + value: "stdoutfile" + - name: CNSLogDir + value: /k + - name: CNSStoreFilePath + value: /k/ + - name: CNS_CONFIGURATION_PATH + value: etc/azure-cns/cns_config.json + - name: NODENAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + hostNetwork: true + volumes: + - name: log + hostPath: + path: /k + type: Directory + - name: cns-config + configMap: + name: cns-config + nodeSelector: + kubernetes.io/os: windows + serviceAccountName: azure-cns +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: cns-config + namespace: kube-system +data: + cns_config.json: | + { + "TelemetrySettings": { + "TelemetryBatchSizeBytes": 16384, + "TelemetryBatchIntervalInSecs": 15, + "RefreshIntervalInSecs": 15, + "DisableAll": false, + "HeartBeatIntervalInMins": 30, + "DebugMode": false, + "SnapshotIntervalInMins": 60 + }, + "ManagedSettings": { + "PrivateEndpoint": "", + "InfrastructureNetworkID": "", + "NodeID": "", + "NodeSyncIntervalInSeconds": 30 + }, + "ChannelMode": "CRD", + "InitializeFromCNI": true + } \ No newline at end of file diff --git a/test/integration/manifests/datapath/linux-deployment-ipv6.yaml b/test/integration/manifests/datapath/linux-deployment-ipv6.yaml new file mode 100644 index 0000000000..32a5ef626a --- /dev/null +++ b/test/integration/manifests/datapath/linux-deployment-ipv6.yaml @@ -0,0 +1,88 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: goldpinger-deploy + namespace: default +spec: + replicas: 4 + selector: + matchLabels: + app: goldpinger + template: + metadata: + labels: + app: goldpinger + spec: + containers: + - name: goldpinger + env: + - name: HOST + value: "0.0.0.0" + - name: PORT + value: "8080" + - name: PING_TIMEOUT + value: "10s" + - name: CHECK_TIMEOUT + value: "20s" + - name: CHECK_ALL_TIMEOUT + value: "20s" + - name: DNS_TARGETS_TIMEOUT + value: "10s" + - name: IP_VERSIONS + value: "6" + - name: HOSTNAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: HOSTS_TO_RESOLVE + value: "2001:4860:4860::8888 www.bing.com" + image: "docker.io/bloomberg/goldpinger:v3.7.0" + serviceAccount: goldpinger-serviceaccount + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + securityContext: + allowPrivilegeEscalation: false + securityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 2000 + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: "app" + operator: In + values: + - goldpinger + topologyKey: "kubernetes.io/hostname" + resources: + limits: + memory: 80Mi + requests: + cpu: 1m + memory: 40Mi + ports: + - containerPort: 8080 + name: http + readinessProbe: + httpGet: + path: /healthz + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 5 + livenessProbe: + httpGet: + path: /healthz + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 5 + nodeSelector: + kubernetes.io/os: linux diff --git a/test/integration/manifests/datapath/linux-deployment.yaml b/test/integration/manifests/datapath/linux-deployment.yaml new file mode 100644 index 0000000000..24ddb5f8c5 --- /dev/null +++ b/test/integration/manifests/datapath/linux-deployment.yaml @@ -0,0 +1,86 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: goldpinger-deploy + namespace: default +spec: + replicas: 4 + selector: + matchLabels: + app: goldpinger + template: + metadata: + labels: + app: goldpinger + spec: + containers: + - name: goldpinger + env: + - name: HOST + value: "0.0.0.0" + - name: PORT + value: "8080" + - name: PING_TIMEOUT + value: "10s" + - name: CHECK_TIMEOUT + value: "20s" + - name: CHECK_ALL_TIMEOUT + value: "20s" + - name: DNS_TARGETS_TIMEOUT + value: "10s" + - name: HOSTNAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: HOSTS_TO_RESOLVE + value: "1.1.1.1 8.8.8.8 www.bing.com" + image: "docker.io/bloomberg/goldpinger:v3.7.0" + serviceAccount: goldpinger-serviceaccount + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + securityContext: + allowPrivilegeEscalation: false + securityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 2000 + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: "app" + operator: In + values: + - goldpinger + topologyKey: "kubernetes.io/hostname" + resources: + limits: + memory: 80Mi + requests: + cpu: 1m + memory: 40Mi + ports: + - containerPort: 8080 + name: http + readinessProbe: + httpGet: + path: /healthz + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 5 + livenessProbe: + httpGet: + path: /healthz + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 5 + nodeSelector: + kubernetes.io/os: linux diff --git a/test/integration/manifests/goldpinger/cluster-role-binding.yaml b/test/integration/manifests/goldpinger/cluster-role-binding.yaml index c7c22e9bb3..e18b186a12 100644 --- a/test/integration/manifests/goldpinger/cluster-role-binding.yaml +++ b/test/integration/manifests/goldpinger/cluster-role-binding.yaml @@ -1,4 +1,4 @@ -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: goldpinger-clusterrolebinding diff --git a/test/integration/manifests/goldpinger/daemonset-ipv6.yaml b/test/integration/manifests/goldpinger/daemonset-ipv6.yaml new file mode 100644 index 0000000000..cf93e09ae3 --- /dev/null +++ b/test/integration/manifests/goldpinger/daemonset-ipv6.yaml @@ -0,0 +1,82 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: goldpinger-host + namespace: default +spec: + selector: + matchLabels: + app: goldpinger + type: goldpinger-host + template: + metadata: + labels: + app: goldpinger + type: goldpinger-host + spec: + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + hostNetwork: true + serviceAccount: "goldpinger-serviceaccount" + securityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 2000 + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: "app" + operator: In + values: + - goldpinger + topologyKey: "kubernetes.io/hostname" + containers: + - name: goldpinger-vm + env: + - name: HOST + value: "0.0.0.0" + - name: PORT + value: "8080" + - name: PING_TIMEOUT + value: "10s" + - name: CHECK_TIMEOUT + value: "20s" + - name: CHECK_ALL_TIMEOUT + value: "20s" + - name: IP_VERSIONS + value: "6" + - name: HOSTNAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: HOSTS_TO_RESOLVE + value: "2001:4860:4860::8888 www.bing.com" + image: "docker.io/bloomberg/goldpinger:v3.7.0" + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + ports: + - containerPort: 8080 + name: http + readinessProbe: + httpGet: + path: /healthz + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 5 + livenessProbe: + httpGet: + path: /healthz + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 5 diff --git a/test/integration/manifests/load/privileged-daemonset.yaml b/test/integration/manifests/load/privileged-daemonset.yaml index 9bacdc4ebe..45398306d5 100644 --- a/test/integration/manifests/load/privileged-daemonset.yaml +++ b/test/integration/manifests/load/privileged-daemonset.yaml @@ -26,13 +26,25 @@ spec: volumeMounts: - mountPath: /var/run/azure-cns name: azure-cns + - mountPath: /var/run/azure-network + name: azure-network - mountPath: /host name: host-root + - mountPath: /var/run + name: azure-cns-noncilium volumes: - name: azure-cns hostPath: path: /var/run/azure-cns + - name: azure-network + hostPath: + path: /var/run/azure-network + - name: azure-cns-noncilium + hostPath: + path: /var/run - hostPath: path: / type: "" name: host-root + nodeSelector: + kubernetes.io/os: linux diff --git a/test/integration/manifests/noop-deployment-linux.yaml b/test/integration/manifests/noop-deployment-linux.yaml index 6b12793189..4d4acd89c2 100644 --- a/test/integration/manifests/noop-deployment-linux.yaml +++ b/test/integration/manifests/noop-deployment-linux.yaml @@ -20,4 +20,4 @@ spec: securityContext: privileged: true nodeSelector: - "kubernetes.io/os": linux + kubernetes.io/os: linux diff --git a/test/integration/manifests/noop-deployment-windows.yaml b/test/integration/manifests/noop-deployment-windows.yaml index 3b35f044dc..7d6f5ef035 100644 --- a/test/integration/manifests/noop-deployment-windows.yaml +++ b/test/integration/manifests/noop-deployment-windows.yaml @@ -20,4 +20,4 @@ spec: ports: - containerPort: 80 nodeSelector: - "kubernetes.io/os": windows + kubernetes.io/os: windows diff --git a/test/integration/setup_test.go b/test/integration/setup_test.go index 5ece6e6498..ff67f7c784 100644 --- a/test/integration/setup_test.go +++ b/test/integration/setup_test.go @@ -17,13 +17,14 @@ import ( const ( exitFail = 1 - envTestDropgz = "TEST_DROPGZ" - envCNIDropgzVersion = "CNI_DROPGZ_VERSION" - envCNSVersion = "CNS_VERSION" - envInstallCNS = "INSTALL_CNS" - envInstallAzilium = "INSTALL_AZILIUM" - envInstallAzureVnet = "INSTALL_AZURE_VNET" - envInstallOverlay = "INSTALL_OVERLAY" + envTestDropgz = "TEST_DROPGZ" + envCNIDropgzVersion = "CNI_DROPGZ_VERSION" + envCNSVersion = "CNS_VERSION" + envInstallCNS = "INSTALL_CNS" + envInstallAzilium = "INSTALL_AZILIUM" + envInstallAzureVnet = "INSTALL_AZURE_VNET" + envInstallOverlay = "INSTALL_OVERLAY" + envInstallDualStackOverlay = "INSTALL_DUALSTACK_OVERLAY" // relative cns manifest paths cnsManifestFolder = "manifests/cns" @@ -160,6 +161,19 @@ func installCNSDaemonset(ctx context.Context, clientset *kubernetes.Clientset, l log.Printf("Env %v not set to true, skipping", envInstallOverlay) } + if installBool4 := os.Getenv(envInstallDualStackOverlay); installBool4 != "" { + if dualStackOverlayScenario, err := strconv.ParseBool(installBool4); err == nil && dualStackOverlayScenario == true { + log.Printf("Env %v set to true, deploy azure-vnet", envInstallDualStackOverlay) + cns.Spec.Template.Spec.InitContainers[0].Args = []string{"deploy", "azure-vnet", "-o", "/opt/cni/bin/azure-vnet", "azure-vnet-telemetry", "-o", "/opt/cni/bin/azure-vnet-telemetry", "azure-vnet-ipam", "-o", "/opt/cni/bin/azure-vnet-ipam", "azure-swift-overlay-dualstack.conflist", "-o", "/etc/cni/net.d/10-azure.conflist"} + } + // setup the CNS swiftconfigmap + if err := k8sutils.MustSetupConfigMap(ctx, clientset, cnsSwiftConfigMapPath); err != nil { + return nil, err + } + } else { + log.Printf("Env %v not set to true, skipping", envInstallDualStackOverlay) + } + cnsDaemonsetClient := clientset.AppsV1().DaemonSets(cns.Namespace) log.Printf("Installing CNS with image %s", cns.Spec.Template.Spec.Containers[0].Image) diff --git a/test/internal/datapath/datapath_linux.go b/test/internal/datapath/datapath_linux.go new file mode 100644 index 0000000000..d4db62b15d --- /dev/null +++ b/test/internal/datapath/datapath_linux.go @@ -0,0 +1 @@ +package datapath diff --git a/test/internal/datapath/datapath_win.go b/test/internal/datapath/datapath_win.go index 54a317760b..d48084af70 100644 --- a/test/internal/datapath/datapath_win.go +++ b/test/internal/datapath/datapath_win.go @@ -3,6 +3,7 @@ package datapath import ( "context" "fmt" + "net" "strings" "github.com/Azure/azure-container-networking/test/internal/k8sutils" @@ -14,6 +15,8 @@ import ( restclient "k8s.io/client-go/rest" ) +var ipv6PrefixPolicy = []string{"curl", "-6", "-I", "-v", "www.bing.com"} + func podTest(ctx context.Context, clientset *kubernetes.Clientset, srcPod *apiv1.Pod, cmd []string, rc *restclient.Config, passFunc func(string) error) error { logrus.Infof("podTest() - %v %v", srcPod.Name, cmd) output, err := k8sutils.ExecCmdOnPod(ctx, clientset, srcPod.Namespace, srcPod.Name, cmd, rc) @@ -48,8 +51,27 @@ func WindowsPodToPodPingTestSameNode(ctx context.Context, clientset *kubernetes. } logrus.Infof("Second pod: %v %v", secondPod.Name, secondPod.Status.PodIP) + // ipv4 ping test // Ping the second pod from the first pod - return podTest(ctx, clientset, firstPod, []string{"ping", secondPod.Status.PodIP}, rc, pingPassedWindows) + resultOne := podTest(ctx, clientset, firstPod, []string{"ping", secondPod.Status.PodIP}, rc, pingPassedWindows) + if resultOne != nil { + return resultOne + } + + // ipv6 ping test + // ipv6 Ping the second pod from the first pod + if len(secondPod.Status.PodIPs) > 1 { + for _, ip := range secondPod.Status.PodIPs { + if net.ParseIP(ip.IP).To16() != nil { + resultTwo := podTest(ctx, clientset, firstPod, []string{"ping", ip.IP}, rc, pingPassedWindows) + if resultTwo != nil { + return resultTwo + } + } + } + } + + return nil } func WindowsPodToPodPingTestDiffNode(ctx context.Context, clientset *kubernetes.Clientset, nodeName1, nodeName2, podNamespace, labelSelector string, rc *restclient.Config) error { @@ -80,7 +102,23 @@ func WindowsPodToPodPingTestDiffNode(ctx context.Context, clientset *kubernetes. logrus.Infof("Second pod: %v %v", secondPod.Name, secondPod.Status.PodIP) // Ping the second pod from the first pod located on different nodes - return podTest(ctx, clientset, firstPod, []string{"ping", secondPod.Status.PodIP}, rc, pingPassedWindows) + resultOne := podTest(ctx, clientset, firstPod, []string{"ping", secondPod.Status.PodIP}, rc, pingPassedWindows) + if resultOne != nil { + return resultOne + } + + if len(secondPod.Status.PodIPs) > 1 { + for _, ip := range secondPod.Status.PodIPs { + if net.ParseIP(ip.IP).To16() != nil { + resultTwo := podTest(ctx, clientset, firstPod, []string{"ping ", ip.IP}, rc, pingPassedWindows) + if resultTwo != nil { + return resultTwo + } + } + } + } + + return nil } func WindowsPodToNode(ctx context.Context, clientset *kubernetes.Clientset, nodeName, nodeIP, podNamespace, labelSelector string, rc *restclient.Config) error { @@ -108,6 +146,7 @@ func WindowsPodToNode(ctx context.Context, clientset *kubernetes.Clientset, node logrus.Infof("Second pod: %v", secondPod.Name) // Ping from pod to node + logrus.Infof("Node IP: %s", nodeIP) resultOne := podTest(ctx, clientset, firstPod, []string{"ping", nodeIP}, rc, pingPassedWindows) resultTwo := podTest(ctx, clientset, secondPod, []string{"ping", nodeIP}, rc, pingPassedWindows) @@ -158,6 +197,22 @@ func WindowsPodToInternet(ctx context.Context, clientset *kubernetes.Clientset, return resultTwo } + // test Invoke-WebRequest an URL by IPv6 address on one pod + // command is: C:\inetpub\wwwroot>curl -6 -I -v www.bing.com + // then return * Trying [2620:1ec:c11::200]:80... + // HTTP/1.1 200 OK + if len(secondPod.Status.PodIPs) > 1 { + for _, ip := range secondPod.Status.PodIPs { + logrus.Infof("pods.Items[0].Name is %s", pods.Items[0].Name) + if net.ParseIP(ip.IP).To16() != nil { + resultThree := podTest(ctx, clientset, secondPod, ipv6PrefixPolicy, rc, webRequestPassedWindows) + if resultThree != nil { + return resultThree + } + } + } + } + return nil } diff --git a/test/internal/k8sutils/utils_get.go b/test/internal/k8sutils/utils_get.go index 531ec38fce..6c1ff2b0e6 100644 --- a/test/internal/k8sutils/utils_get.go +++ b/test/internal/k8sutils/utils_get.go @@ -43,9 +43,11 @@ func GetPodsIpsByNode(ctx context.Context, clientset *kubernetes.Clientset, name if err != nil { return nil, err } - ips := make([]string, 0, len(pods.Items)) + ips := make([]string, 0, len(pods.Items)*2) //nolint for index := range pods.Items { - ips = append(ips, pods.Items[index].Status.PodIP) + for _, podIP := range pods.Items[index].Status.PodIPs { + ips = append(ips, podIP.IP) + } } return ips, nil } diff --git a/test/validate/client.go b/test/validate/client.go index b4cc5f5cb8..0ede9bbb5c 100644 --- a/test/validate/client.go +++ b/test/validate/client.go @@ -20,6 +20,8 @@ type Validator struct { type IValidator interface { ValidateStateFile() error ValidateRestartNetwork() error + ValidateDualStackNodeProperties() error + // ValidateDataPath() error } diff --git a/test/validate/linux_validate.go b/test/validate/linux_validate.go index d2839f4098..e38e6531e8 100644 --- a/test/validate/linux_validate.go +++ b/test/validate/linux_validate.go @@ -9,6 +9,7 @@ import ( restserver "github.com/Azure/azure-container-networking/cns/restserver" k8sutils "github.com/Azure/azure-container-networking/test/internal/k8sutils" "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" ) @@ -18,17 +19,25 @@ const ( privilegedLabelSelector = "app=privileged-daemonset" privilegedNamespace = "kube-system" - cnsLabelSelector = "k8s-app=azure-cns" - ciliumLabelSelector = "k8s-app=cilium" + cnsLabelSelector = "k8s-app=azure-cns" + ciliumLabelSelector = "k8s-app=cilium" + overlayClusterLabelName = "overlay" + dualstackNodeStatusAddr = 3 ) var ( - restartNetworkCmd = []string{"bash", "-c", "chroot /host /bin/bash -c 'systemctl restart systemd-networkd'"} - cnsStateFileCmd = []string{"bash", "-c", "cat /var/run/azure-cns/azure-endpoints.json"} - ciliumStateFileCmd = []string{"bash", "-c", "cilium endpoint list -o json"} - cnsLocalCacheCmd = []string{"curl", "localhost:10090/debug/ipaddresses", "-d", "{\"IPConfigStateFilter\":[\"Assigned\"]}"} + restartNetworkCmd = []string{"bash", "-c", "chroot /host /bin/bash -c 'systemctl restart systemd-networkd'"} + cnsStateFileCmd = []string{"bash", "-c", "cat /var/run/azure-cns/azure-endpoints.json"} + azureCnsStateFileCmd = []string{"bash", "-c", "cat /var/run/azure-vnet.json"} // azure cni statefile is /var/run/azure-vnet.json + ciliumStateFileCmd = []string{"bash", "-c", "cilium endpoint list -o json"} + cnsLocalCacheCmd = []string{"curl", "localhost:10090/debug/ipaddresses", "-d", "{\"IPConfigStateFilter\":[\"Assigned\"]}"} ) +var dualstackoverlaynodelabel = map[string]string{ + "kubernetes.azure.com/podnetwork-type": "overlay", + "kubernetes.azure.com/podv6network-type": "overlay", +} + type stateFileIpsFunc func([]byte) (map[string]string, error) type LinuxClient struct{} @@ -62,6 +71,63 @@ type Address struct { Addr string `json:"ipv4"` } +// parse azure-vnet.json +// azure cni manages endpoint state +type AzureCniState struct { + AzureCniState AzureVnetNetwork `json:"Network"` +} + +type AzureVnetNetwork struct { + Version string `json:"Version"` + TimeStamp string `json:"TimeStamp"` + ExternalInterfaces map[string]InterfaceInfo `json:"ExternalInterfaces"` // key: interface name; value: Interface Info +} + +type InterfaceInfo struct { + Name string `json:"Name"` + Networks map[string]AzureVnetNetworkInfo `json:"Networks"` // key: networkName, value: AzureVnetNetworkInfo +} + +type AzureVnetInfo struct { + Name string + Networks map[string]AzureVnetNetworkInfo // key: network name, value: NetworkInfo +} + +type AzureVnetNetworkInfo struct { + ID string + Mode string + Subnets []Subnet + Endpoints map[string]AzureVnetEndpointInfo // key: azure endpoint name, value: AzureVnetEndpointInfo + PODName string +} + +type Subnet struct { + Family int + Prefix Prefix + Gateway string + PrimaryIP string +} + +type Prefix struct { + IP string + Mask string +} + +type AzureVnetEndpointInfo struct { + IfName string + MacAddress string + IPAddresses []Prefix + PODName string +} + +type check struct { + name string + stateFileIps func([]byte) (map[string]string, error) + podLabelSelector string + podNamespace string + cmd []string +} + func (l *LinuxClient) CreateClient(ctx context.Context, clienset *kubernetes.Clientset, config *rest.Config, namespace, cni string, restartCase bool) IValidator { // deploy privileged pod privilegedDaemonSet, err := k8sutils.MustParseDaemonSet(privilegedDaemonSetPath) @@ -89,21 +155,22 @@ func (l *LinuxClient) CreateClient(ctx context.Context, clienset *kubernetes.Cli } } -// Todo: Based on cni version validate different state files func (v *LinuxValidator) ValidateStateFile() error { - checks := []struct { - name string - stateFileIps func([]byte) (map[string]string, error) - podLabelSelector string - podNamespace string - cmd []string - }{ + checkSet := make(map[string][]check) // key is cni type, value is a list of check + + // TODO: add cniv1 when adding Linux related test cases + checkSet["cilium"] = []check{ {"cns", cnsStateFileIps, cnsLabelSelector, privilegedNamespace, cnsStateFileCmd}, {"cilium", ciliumStateFileIps, ciliumLabelSelector, privilegedNamespace, ciliumStateFileCmd}, {"cns cache", cnsCacheStateFileIps, cnsLabelSelector, privilegedNamespace, cnsLocalCacheCmd}, } - for _, check := range checks { + checkSet["cniv2"] = []check{ + {"cns cache", cnsCacheStateFileIps, cnsLabelSelector, privilegedNamespace, cnsLocalCacheCmd}, + {"azure dualstackoverlay", azureDualStackStateFileIPs, privilegedLabelSelector, privilegedNamespace, azureCnsStateFileCmd}, + } + + for _, check := range checkSet[v.cni] { err := v.validate(check.stateFileIps, check.cmd, check.name, check.podNamespace, check.podLabelSelector) if err != nil { return err @@ -158,6 +225,31 @@ func cnsStateFileIps(result []byte) (map[string]string, error) { return cnsPodIps, nil } +func azureDualStackStateFileIPs(result []byte) (map[string]string, error) { + var azureDualStackResult AzureCniState + err := json.Unmarshal(result, &azureDualStackResult) + if err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal azure cni endpoint list") + } + + azureCnsPodIps := make(map[string]string) + for _, v := range azureDualStackResult.AzureCniState.ExternalInterfaces { + for _, networks := range v.Networks { + for _, ip := range networks.Endpoints { + pod := ip.PODName + ipv4 := ip.IPAddresses[0].IP + azureCnsPodIps[ipv4] = pod + + if len(ip.IPAddresses) > 1 { + ipv6 := ip.IPAddresses[1].IP + azureCnsPodIps[ipv6] = pod + } + } + } + } + return azureCnsPodIps, nil +} + func ciliumStateFileIps(result []byte) (map[string]string, error) { var ciliumResult []CiliumEndpointStatus err := json.Unmarshal(result, &ciliumResult) @@ -230,3 +322,39 @@ func (v *LinuxValidator) validate(stateFileIps stateFileIpsFunc, cmd []string, c log.Printf("State file validation for %s passed", checkType) return nil } + +func (v *LinuxValidator) ValidateDualStackNodeProperties() error { + log.Print("Validating Dualstack Overlay Linux Node properties") + nodes, err := k8sutils.GetNodeList(v.ctx, v.clientset) + if err != nil { + return errors.Wrapf(err, "failed to get node list") + } + + for index := range nodes.Items { + nodeName := nodes.Items[index].ObjectMeta.Name + // check node status + nodeConditions := nodes.Items[index].Status.Conditions + if nodeConditions[len(nodeConditions)-1].Type != corev1.NodeReady { + return errors.Wrapf(err, "node %s status is not ready", nodeName) + } + + // get node labels + nodeLabels := nodes.Items[index].ObjectMeta.GetLabels() + for key := range nodeLabels { + if value, ok := dualstackoverlaynodelabel[key]; ok { + log.Printf("label %s is correctly shown on the node %+v", key, nodeName) + if value != overlayClusterLabelName { + return errors.Wrapf(err, "node %s overlay label name is wrong", nodeName) + } + } + } + + // get node allocated IPs and check whether it includes ipv4 and ipv6 address + // node status addresses object will return three objects; two of them are ip addresses object(one is ipv4 and one is ipv6) + if len(nodes.Items[index].Status.Addresses) < dualstackNodeStatusAddr { + return errors.Wrapf(err, "node %s is missing IPv6 internal IP", nodeName) + } + } + + return nil +} diff --git a/test/validate/utils.go b/test/validate/utils.go index 7180c7bc66..4c81fe145a 100644 --- a/test/validate/utils.go +++ b/test/validate/utils.go @@ -2,6 +2,7 @@ package validate import ( "context" + "reflect" "github.com/Azure/azure-container-networking/test/internal/k8sutils" corev1 "k8s.io/api/core/v1" @@ -29,11 +30,30 @@ func getPodIPsWithoutNodeIP(ctx context.Context, clientset *kubernetes.Clientset if err != nil { return podsIpsWithoutNodeIP } - nodeIP := node.Status.Addresses[0].Address + nodeIPs := make([]string, 0) + for _, address := range node.Status.Addresses { + if address.Type == corev1.NodeInternalIP { + nodeIPs = append(nodeIPs, address.Address) + } + } + for _, podIP := range podIPs { - if podIP != nodeIP { + if !contain(podIP, nodeIPs) { podsIpsWithoutNodeIP = append(podsIpsWithoutNodeIP, podIP) } } return podsIpsWithoutNodeIP } + +func contain(obj, target interface{}) bool { + targetValue := reflect.ValueOf(target) + switch reflect.TypeOf(target).Kind() { //nolint + case reflect.Slice, reflect.Array: + for i := 0; i < targetValue.Len(); i++ { + if targetValue.Index(i).Interface() == obj { + return true + } + } + } + return false +} diff --git a/test/validate/windows_validate.go b/test/validate/windows_validate.go index 9e54f61bef..b7dafa3032 100644 --- a/test/validate/windows_validate.go +++ b/test/validate/windows_validate.go @@ -8,6 +8,7 @@ import ( k8sutils "github.com/Azure/azure-container-networking/test/internal/k8sutils" "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" ) @@ -18,7 +19,8 @@ const ( ) var ( - hnsEndPpointCmd = []string{"powershell", "-c", "Get-HnsEndpoint | ConvertTo-Json"} + hnsEndPointCmd = []string{"powershell", "-c", "Get-HnsEndpoint | ConvertTo-Json"} + hnsNetworkCmd = []string{"powershell", "-c", "Get-HnsNetwork | ConvertTo-Json"} azureVnetCmd = []string{"powershell", "-c", "cat ../../k/azure-vnet.json"} azureVnetIpamCmd = []string{"powershell", "-c", "cat ../../k/azure-vnet-ipam.json"} ) @@ -78,6 +80,14 @@ type AddressRecord struct { InUse bool } +type HNSNetwork struct { + Name string `json:"Name"` + IPv6 bool `json:"IPv6"` + ManagementIP string `json:"ManagementIP"` + ManagementIPv6 string `json:"ManagementIPv6"` + State int `json:"State"` +} + func (w *WindowsClient) CreateClient(ctx context.Context, clienset *kubernetes.Clientset, config *rest.Config, namespace, cni string, restartCase bool) IValidator { // deploy privileged pod privilegedDaemonSet, err := k8sutils.MustParseDaemonSet(privilegedWindowsDaemonSetPath) @@ -106,24 +116,26 @@ func (w *WindowsClient) CreateClient(ctx context.Context, clienset *kubernetes.C } func (v *WindowsValidator) ValidateStateFile() error { - checks := []struct { - name string - stateFileIps func([]byte) (map[string]string, error) - podLabelSelector string - podNamespace string - cmd []string - }{ - {"hns", hnsStateFileIps, privilegedLabelSelector, privilegedNamespace, hnsEndPpointCmd}, + checkSet := make(map[string][]check) // key is cni type, value is a list of check + + checkSet["cniv1"] = []check{ + {"hns", hnsStateFileIps, privilegedLabelSelector, privilegedNamespace, hnsEndPointCmd}, {"azure-vnet", azureVnetIps, privilegedLabelSelector, privilegedNamespace, azureVnetCmd}, {"azure-vnet-ipam", azureVnetIpamIps, privilegedLabelSelector, privilegedNamespace, azureVnetIpamCmd}, } - for _, check := range checks { - err := v.validate(check.stateFileIps, check.cmd, check.name, check.podNamespace, check.podLabelSelector) + checkSet["cniv2"] = []check{ + {"azure-vnet", azureVnetIps, privilegedLabelSelector, privilegedNamespace, azureVnetCmd}, + } + + // this is checking all IPs of the pods with the statefile + for _, check := range checkSet[v.cni] { + err := v.validateIPs(check.stateFileIps, check.cmd, check.name, check.podNamespace, check.podLabelSelector) if err != nil { return err } } + return nil } @@ -140,9 +152,21 @@ func hnsStateFileIps(result []byte) (map[string]string, error) { hnsPodIps[v.IPAddress.String()] = v.MacAddress } } + return hnsPodIps, nil } +// return windows HNS network state +func hnsNetworkState(result []byte) ([]HNSNetwork, error) { + var hnsNetworkResult []HNSNetwork + err := json.Unmarshal(result, &hnsNetworkResult) + if err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal hns network list") + } + + return hnsNetworkResult, nil +} + func azureVnetIps(result []byte) (map[string]string, error) { var azureVnetResult AzureVnet err := json.Unmarshal(result, &azureVnetResult) @@ -155,11 +179,13 @@ func azureVnetIps(result []byte) (map[string]string, error) { for _, v := range v.Networks { for _, e := range v.Endpoints { for _, v := range e.IPAddresses { + // collect both ipv4 and ipv6 addresses azureVnetPodIps[v.IP.String()] = e.IfName } } } } + return azureVnetPodIps, nil } @@ -184,7 +210,7 @@ func azureVnetIpamIps(result []byte) (map[string]string, error) { return azureVnetIpamPodIps, nil } -func (v *WindowsValidator) validate(stateFileIps stateFileIpsFunc, cmd []string, checkType, namespace, labelSelector string) error { +func (v *WindowsValidator) validateIPs(stateFileIps stateFileIpsFunc, cmd []string, checkType, namespace, labelSelector string) error { log.Println("Validating ", checkType, " state file") nodes, err := k8sutils.GetNodeListByLabelSelector(v.ctx, v.clientset, windowsNodeSelector) if err != nil { @@ -226,3 +252,69 @@ func (v *WindowsValidator) validate(stateFileIps stateFileIpsFunc, cmd []string, func (v *WindowsValidator) ValidateRestartNetwork() error { return nil } + +func (v *WindowsValidator) ValidateDualStackNodeProperties() error { + log.Print("Validating Dualstack Overlay Windows Node properties") + nodes, err := k8sutils.GetNodeListByLabelSelector(v.ctx, v.clientset, windowsNodeSelector) + if err != nil { + return errors.Wrapf(err, "failed to get node list") + } + + for index := range nodes.Items { + nodeName := nodes.Items[index].ObjectMeta.Name + // check node status + nodeConditions := nodes.Items[index].Status.Conditions + if nodeConditions[len(nodeConditions)-1].Type != corev1.NodeReady { + return errors.Wrapf(err, "node %s status is not ready", nodeName) + } + + // get node labels + nodeLabels := nodes.Items[index].ObjectMeta.GetLabels() + for key := range nodeLabels { + if value, ok := dualstackoverlaynodelabel[key]; ok { + log.Printf("label %s is correctly shown on the node %+v", key, nodeName) + if value != overlayClusterLabelName { + return errors.Wrapf(err, "node %s overlay label name is wrong", nodeName) + } + } + } + + // check windows HNS network state + pod, err := k8sutils.GetPodsByNode(v.ctx, v.clientset, privilegedNamespace, privilegedLabelSelector, nodes.Items[index].Name) + if err != nil { + return errors.Wrapf(err, "failed to get privileged pod") + } + + podName := pod.Items[0].Name + // exec into the pod to get the state file + result, err := k8sutils.ExecCmdOnPod(v.ctx, v.clientset, privilegedNamespace, podName, hnsNetworkCmd, v.config) + if err != nil { + return errors.Wrapf(err, "failed to exec into privileged pod") + } + + hnsNetwork, err := hnsNetworkState(result) + if err != nil { + return errors.Wrapf(err, "failed to unmarshal hns network list") + } + + if len(hnsNetwork) == 0 { //nolint + return errors.Wrapf(err, "windows node does not have any HNS network") + } else if len(hnsNetwork) == 1 { //nolint + return errors.Wrapf(err, "HNS default ext network or azure network does not exist") + } else { + for _, network := range hnsNetwork { + if !network.IPv6 { + return errors.Wrapf(err, "windows HNS network IPv6 flag is not set correctly") + } + if network.State != 1 { + return errors.Wrapf(err, "windows HNS network state is not correct") + } + if network.ManagementIPv6 == "" || network.ManagementIP == "" { + return errors.Wrapf(err, "windows HNS network is missing ipv4 or ipv6 management IP") + } + } + } + } + + return nil +}