Aks swift test update

.gitignore

@@ -13,4 +13,7 @@ ipam-*.xml
 *.out
 
 # Goland
-.idea/*
+.idea/*
+
+# Logs
+*.log

.pipelines/singletenancy/aks-swift/e2e-job-template.yaml

@@ -23,6 +23,7 @@ stages:
           GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path
           GOBIN: "$(GOPATH)/bin" # Go binaries path
           ImageTag: $[ stagedependencies.build_and_test.unit_tests.outputs['EnvironmentalVariables.ImageTag'] ]
+          modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking"
         steps:
           - template: e2e-step-template.yaml
             parameters:

.pipelines/singletenancy/aks-swift/e2e-step-template.yaml

@@ -31,16 +31,45 @@ steps:
       addSpnToEnvironment: true
       inlineScript: |
         mkdir -p ~/.kube/
-        az keyvault secret show --name $(KV_AKS_SWIFT_SECRET_NAME) --vault-name $(KV_NAME) --query value | sed 's/\\n/\n/g' | sed -e 's/^"//' -e 's/"$//'
-        az keyvault secret show --name $(KV_AKS_SWIFT_SECRET_NAME) --vault-name $(KV_NAME) --query value | sed 's/\\n/\n/g' | sed -e 's/^"//' -e 's/"$//' > ~/.kube/config
+        echo "DNC Underlay config"
+        az keyvault secret show --name $(KV_DNC_UNDERLAY_SECRET_NAME) --vault-name $(KV_NAME) --query value | sed 's/\\n/\n/g' | sed -e 's/^"//' -e 's/"$//'
+        az keyvault secret show --name $(KV_DNC_UNDERLAY_SECRET_NAME) --vault-name $(KV_NAME) --query value | sed 's/\\n/\n/g' | sed -e 's/^"//' -e 's/"$//' > ~/.kube/dnc-underlay
+        echo "Dirty ACN cx cluster config"
+        az keyvault secret show --name $(KV_AKS_SWIFT_CX_SECRET_NAME) --vault-name $(KV_NAME) --query value | sed 's/\\n/\n/g' | sed -e 's/^"//' -e 's/"$//'
+        az keyvault secret show --name $(KV_AKS_SWIFT_CX_SECRET_NAME) --vault-name $(KV_NAME) --query value | sed 's/\\n/\n/g' | sed -e 's/^"//' -e 's/"$//' > ~/.kube/config
     displayName: Pull Kubeconfig from Keyvault
     condition: succeeded()
 
   - script: |
       ls -lah
       pwd
+      echo "installing kubectl"
+      curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+      sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
+      kubectl cluster-info
+      kubectl get po -owide -A
       sudo -E env "PATH=$PATH" VERSION=$(ImageTag) INSTALL_CNI=true INSTALL_CNS=true make test-integration
     name: "aksswifte2e"
     displayName: "Run AKS Swift E2E"
 
+  - script: |
+      echo "Logs will be available as a build artifact"
+      ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/aks-swift-output/
+      echo $ARTIFACT_DIR
+      sudo rm -rf $ARTIFACT_DIR
+      sudo mkdir $ARTIFACT_DIR
+      sudo cp test/integration/logs/* $ARTIFACT_DIR
+      export KUBECONFIG=~/.kube/dnc-underlay
+      kubectl logs -n $STABLE_DNC_NAMESPACE -l app=dnc --tail -1 -c dnc --prefix > dnc-logs.txt
+      kubectl logs -n $STABLE_DNC_NAMESPACE -l app=dnc --tail -1 -c requestcontroller --prefix > dnc-rc-logs.txt
+
+      sudo cp *.txt $ARTIFACT_DIR
+    name: "GetLogs"
+    displayName: "Get logs"
+    condition: always()
 
+  - task: PublishBuildArtifacts@1
+    inputs:
+      artifactName: aks-swift-output
+      pathtoPublish: "$(Build.ArtifactStagingDirectory)/aks-swift-output"
+    condition: always()

Makefile

@@ -266,7 +266,7 @@ tools: acncli
 
 .PHONY: tools-images
 tools-images: 
-	docker build -f ./tools/acncli/Dockerfile --build-arg VERSION=$(VERSION) -t $(AZURE_CNI_IMAGE):$(VERSION) .
+	docker build --no-cache -f ./tools/acncli/Dockerfile --build-arg VERSION=$(VERSION) -t $(AZURE_CNI_IMAGE):$(VERSION) .
 
 # Build the Azure CNM plugin image, installable with "docker plugin install".
 .PHONY: azure-vnet-plugin-image

test/integration/manifests/cni/manager.yaml

@@ -19,7 +19,9 @@ spec:
       hostNetwork: true
       containers:
         - name: azure-cni-installer
-          image: mcr.microsoft.com/containernetworking/azure-cni-manager:v1.2.0-2-g0671b63 
+          image: acnpublic.azurecr.io/azure-cni-manager:v1.2.8-32-g77506640
+          command: ["./acn"]
+          args: ["cni", "manager", "--follow", "--mode", "transparent", "--ipam", "azure-cns"]
           imagePullPolicy: Always
           env:
             - name: AZURE_CNI_OS

test/integration/manifests/cns/daemonset.yaml

@@ -41,7 +41,7 @@ spec:
           effect: NoSchedule
       containers:
         - name: cns-container
-          image: mcr.microsoft.com/containernetworking/azure-cns:v1.2.0
+          image: acnpublic.azurecr.io/azure-cns:v1.2.8
           imagePullPolicy: IfNotPresent
           args: [ "-c", "tcp://$(CNSIpAddress):$(CNSPort)", "-t", "$(CNSLogTarget)"]
           volumeMounts:

test/integration/setup_test.go

@@ -36,6 +36,9 @@ const (
 	cnsRoleBindingPath        = cnsManifestFolder + "/rolebinding.yaml"
 	cnsServiceAccountPath     = cnsManifestFolder + "/serviceaccount.yaml"
 	cnsLabelSelector          = "k8s-app=azure-cns"
+
+	// relative log directory
+	logDir = "logs/"
 )
 
 func TestMain(m *testing.M) {
@@ -75,8 +78,8 @@ func TestMain(m *testing.M) {
 	ctx := context.Background()
 	if installopt := os.Getenv(envInstallCNI); installopt != "" {
 		// create dirty cni-manager ds
-		if installCNI, err := strconv.ParseBool(installopt); err != nil && installCNI == true {
-			if cnicleanup, err = installCNIManagerDaemonset(ctx, clientset, os.Getenv(envImageTag)); err != nil {
+		if installCNI, err := strconv.ParseBool(installopt); err == nil && installCNI == true {
+			if cnicleanup, err = installCNIManagerDaemonset(ctx, clientset, os.Getenv(envImageTag), logDir); err != nil {
 				log.Print(err)
 				return
 			}
@@ -87,8 +90,8 @@ func TestMain(m *testing.M) {
 
 	if installopt := os.Getenv(envInstallCNS); installopt != "" {
 		// create dirty cns ds
-		if installCNS, err := strconv.ParseBool(installopt); err != nil && installCNS == true {
-			if cnscleanup, err = installCNSDaemonset(ctx, clientset, os.Getenv(envImageTag)); err != nil {
+		if installCNS, err := strconv.ParseBool(installopt); err == nil && installCNS == true {
+			if cnscleanup, err = installCNSDaemonset(ctx, clientset, os.Getenv(envImageTag), logDir); err != nil {
 				return
 			}
 		}
@@ -99,7 +102,7 @@ func TestMain(m *testing.M) {
 	exitCode = m.Run()
 }
 
-func installCNSDaemonset(ctx context.Context, clientset *kubernetes.Clientset, imageTag string) (func() error, error) {
+func installCNSDaemonset(ctx context.Context, clientset *kubernetes.Clientset, imageTag, logDir string) (func() error, error) {
 	var (
 		err error
 		cns v1.DaemonSet
@@ -119,7 +122,6 @@ func installCNSDaemonset(ctx context.Context, clientset *kubernetes.Clientset, i
 	cnsDaemonsetClient := clientset.AppsV1().DaemonSets(cns.Namespace)
 
 	log.Printf("Installing CNS with image %s", cns.Spec.Template.Spec.Containers[0].Image)
-
 	// setup the CNS configmap
 	if err := mustSetupConfigMap(ctx, clientset, cnsConfigMapPath); err != nil {
 		return nil, err
@@ -144,6 +146,9 @@ func installCNSDaemonset(ctx context.Context, clientset *kubernetes.Clientset, i
 	}
 
 	cleanupds := func() error {
+		if err := exportLogsByLabelSelector(ctx, clientset, cns.Namespace, cnsLabelSelector, logDir); err != nil {
+			return err
+		}
 		if err := mustDeleteDaemonset(ctx, cnsDaemonsetClient, cns); err != nil {
 			return err
 		}
@@ -153,7 +158,7 @@ func installCNSDaemonset(ctx context.Context, clientset *kubernetes.Clientset, i
 	return cleanupds, nil
 }
 
-func installCNIManagerDaemonset(ctx context.Context, clientset *kubernetes.Clientset, imageTag string) (func() error, error) {
+func installCNIManagerDaemonset(ctx context.Context, clientset *kubernetes.Clientset, imageTag, logDir string) (func() error, error) {
 	var (
 		err error
 		cni v1.DaemonSet
@@ -179,6 +184,9 @@ func installCNIManagerDaemonset(ctx context.Context, clientset *kubernetes.Clien
 	}
 
 	cleanupds := func() error {
+		if err := exportLogsByLabelSelector(ctx, clientset, cni.Namespace, cniLabelSelector, logDir); err != nil {
+			return err
+		}
 		if err := mustDeleteDaemonset(ctx, cniDaemonsetClient, cni); err != nil {
 			return err
 		}

test/integration/utils_test.go

@@ -3,8 +3,10 @@
 package k8s
 
 import (
+	"bytes"
 	"context"
 	"errors"
+	"io"
 	"log"
 	"strings"
 	"time"
@@ -220,6 +222,60 @@ func waitForPodsRunning(ctx context.Context, clientset *kubernetes.Clientset, na
 		return nil
 	}
 
-	retrier := retry.Retrier{Attempts: 10, Delay: 2 * time.Second}
+	retrier := retry.Retrier{Attempts: 10, Delay: 6 * time.Second}
 	return retrier.Do(ctx, checkPodIPsFn)
 }
+
+
+func exportLogsByLabelSelector(ctx context.Context, clientset *kubernetes.Clientset, namespace, labelselector, logDir string) error {
+	podsClient := clientset.CoreV1().Pods(namespace)
+	podLogOpts := corev1.PodLogOptions{}
+	logExtension := ".log"
+	podList, err := podsClient.List(ctx, metav1.ListOptions{LabelSelector: labelselector})
+	if err != nil {
+		return err
+	}
+
+	for _, pod := range podList.Items {
+		req := podsClient.GetLogs(pod.Name, &podLogOpts)
+		podLogs, err := req.Stream(ctx)
+		if err != nil {
+			return err
+		}
+		defer podLogs.Close()
+
+		buf := new(bytes.Buffer)
+		_, err = io.Copy(buf, podLogs)
+		if err != nil {
+			return err
+		}
+		str := buf.String()
+		err = writeToFile(logDir, pod.Name + logExtension, str)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func writeToFile(dir, fileName, str string) error {
+	if _, err := os.Stat(dir); os.IsNotExist(err) { 
+		// your dir does not exist
+		os.MkdirAll(dir, 0666)
+	}
+	// open output file
+	f, err := os.Create(dir + fileName)
+	if err != nil {
+		return err
+	}
+	// close fo on exit and check for its returned error
+	defer func() {
+		if err := f.Close(); err != nil {
+			panic(err)
+		}
+	}()
+
+	// If write went ok then err is nil
+	_, err = f.WriteString(str)
+	return err
+}

tools/acncli/Dockerfile

@@ -19,4 +19,4 @@ ENV AZURE_CNI_TENANCY=singletenancy
 ENV AZURE_CNI_IPAM=azure-cns
 ENV AZURE_CNI_MODE=transparent
 
-CMD ["./acn", "manager", "-f"]
+ENTRYPOINT ["./acn", "cni", "manager", "--follow", "--mode", "transparent"]

tools/acncli/api/constants.go

@@ -82,7 +82,7 @@ var (
 		FlagTenancy:                  Singletenancy,
 		FlagIPAM:                     AzureVNETIPAM,
 		FlagExempt:                   AzureTelemetryBin + "," + AzureTelemetryConfig,
-		FlagMode:                     Bridge,
+		FlagMode:                     Transparent,
 		FlagTarget:                   Local,
 		FlagBinDirectory:             DefaultBinDirLinux,
 		FlagConflistDirectory:        DefaultConflistDirLinux,