diff --git a/.gitignore b/.gitignore index 1a96645186..ac260ece45 100644 --- a/.gitignore +++ b/.gitignore @@ -13,4 +13,7 @@ ipam-*.xml *.out # Goland -.idea/* \ No newline at end of file +.idea/* + +# Logs +*.log \ No newline at end of file diff --git a/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml index 77d45d254e..7974863f98 100644 --- a/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml +++ b/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml @@ -23,6 +23,7 @@ stages: GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path GOBIN: "$(GOPATH)/bin" # Go binaries path ImageTag: $[ stagedependencies.build_and_test.unit_tests.outputs['EnvironmentalVariables.ImageTag'] ] + modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking" steps: - template: e2e-step-template.yaml parameters: diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml index 576a3e6f55..c0479925af 100644 --- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml +++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml @@ -31,16 +31,45 @@ steps: addSpnToEnvironment: true inlineScript: | mkdir -p ~/.kube/ - az keyvault secret show --name $(KV_AKS_SWIFT_SECRET_NAME) --vault-name $(KV_NAME) --query value | sed 's/\\n/\n/g' | sed -e 's/^"//' -e 's/"$//' - az keyvault secret show --name $(KV_AKS_SWIFT_SECRET_NAME) --vault-name $(KV_NAME) --query value | sed 's/\\n/\n/g' | sed -e 's/^"//' -e 's/"$//' > ~/.kube/config + echo "DNC Underlay config" + az keyvault secret show --name $(KV_DNC_UNDERLAY_SECRET_NAME) --vault-name $(KV_NAME) --query value | sed 's/\\n/\n/g' | sed -e 's/^"//' -e 's/"$//' + az keyvault secret show --name $(KV_DNC_UNDERLAY_SECRET_NAME) --vault-name $(KV_NAME) --query value | sed 's/\\n/\n/g' | sed -e 's/^"//' -e 's/"$//' > ~/.kube/dnc-underlay + echo "Dirty ACN cx cluster config" + az keyvault secret show --name $(KV_AKS_SWIFT_CX_SECRET_NAME) --vault-name $(KV_NAME) --query value | sed 's/\\n/\n/g' | sed -e 's/^"//' -e 's/"$//' + az keyvault secret show --name $(KV_AKS_SWIFT_CX_SECRET_NAME) --vault-name $(KV_NAME) --query value | sed 's/\\n/\n/g' | sed -e 's/^"//' -e 's/"$//' > ~/.kube/config displayName: Pull Kubeconfig from Keyvault condition: succeeded() - script: | ls -lah pwd + echo "installing kubectl" + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl + kubectl cluster-info + kubectl get po -owide -A sudo -E env "PATH=$PATH" VERSION=$(ImageTag) INSTALL_CNI=true INSTALL_CNS=true make test-integration name: "aksswifte2e" displayName: "Run AKS Swift E2E" + - script: | + echo "Logs will be available as a build artifact" + ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/aks-swift-output/ + echo $ARTIFACT_DIR + sudo rm -rf $ARTIFACT_DIR + sudo mkdir $ARTIFACT_DIR + sudo cp test/integration/logs/* $ARTIFACT_DIR + export KUBECONFIG=~/.kube/dnc-underlay + kubectl logs -n $STABLE_DNC_NAMESPACE -l app=dnc --tail -1 -c dnc --prefix > dnc-logs.txt + kubectl logs -n $STABLE_DNC_NAMESPACE -l app=dnc --tail -1 -c requestcontroller --prefix > dnc-rc-logs.txt + + sudo cp *.txt $ARTIFACT_DIR + name: "GetLogs" + displayName: "Get logs" + condition: always() + - task: PublishBuildArtifacts@1 + inputs: + artifactName: aks-swift-output + pathtoPublish: "$(Build.ArtifactStagingDirectory)/aks-swift-output" + condition: always() diff --git a/Makefile b/Makefile index 86098b9cd2..9a3fb920f9 100644 --- a/Makefile +++ b/Makefile @@ -266,7 +266,7 @@ tools: acncli .PHONY: tools-images tools-images: - docker build -f ./tools/acncli/Dockerfile --build-arg VERSION=$(VERSION) -t $(AZURE_CNI_IMAGE):$(VERSION) . + docker build --no-cache -f ./tools/acncli/Dockerfile --build-arg VERSION=$(VERSION) -t $(AZURE_CNI_IMAGE):$(VERSION) . # Build the Azure CNM plugin image, installable with "docker plugin install". .PHONY: azure-vnet-plugin-image diff --git a/test/integration/manifests/cni/manager.yaml b/test/integration/manifests/cni/manager.yaml index 979e64cb09..5c9fc19e00 100644 --- a/test/integration/manifests/cni/manager.yaml +++ b/test/integration/manifests/cni/manager.yaml @@ -19,7 +19,9 @@ spec: hostNetwork: true containers: - name: azure-cni-installer - image: mcr.microsoft.com/containernetworking/azure-cni-manager:v1.2.0-2-g0671b63 + image: acnpublic.azurecr.io/azure-cni-manager:v1.2.8-32-g77506640 + command: ["./acn"] + args: ["cni", "manager", "--follow", "--mode", "transparent", "--ipam", "azure-cns"] imagePullPolicy: Always env: - name: AZURE_CNI_OS diff --git a/test/integration/manifests/cns/daemonset.yaml b/test/integration/manifests/cns/daemonset.yaml index b4d1de61bc..ed990549cd 100644 --- a/test/integration/manifests/cns/daemonset.yaml +++ b/test/integration/manifests/cns/daemonset.yaml @@ -41,7 +41,7 @@ spec: effect: NoSchedule containers: - name: cns-container - image: mcr.microsoft.com/containernetworking/azure-cns:v1.2.0 + image: acnpublic.azurecr.io/azure-cns:v1.2.8 imagePullPolicy: IfNotPresent args: [ "-c", "tcp://$(CNSIpAddress):$(CNSPort)", "-t", "$(CNSLogTarget)"] volumeMounts: diff --git a/test/integration/setup_test.go b/test/integration/setup_test.go index 1d118eca10..a48b17a25d 100644 --- a/test/integration/setup_test.go +++ b/test/integration/setup_test.go @@ -36,6 +36,9 @@ const ( cnsRoleBindingPath = cnsManifestFolder + "/rolebinding.yaml" cnsServiceAccountPath = cnsManifestFolder + "/serviceaccount.yaml" cnsLabelSelector = "k8s-app=azure-cns" + + // relative log directory + logDir = "logs/" ) func TestMain(m *testing.M) { @@ -75,8 +78,8 @@ func TestMain(m *testing.M) { ctx := context.Background() if installopt := os.Getenv(envInstallCNI); installopt != "" { // create dirty cni-manager ds - if installCNI, err := strconv.ParseBool(installopt); err != nil && installCNI == true { - if cnicleanup, err = installCNIManagerDaemonset(ctx, clientset, os.Getenv(envImageTag)); err != nil { + if installCNI, err := strconv.ParseBool(installopt); err == nil && installCNI == true { + if cnicleanup, err = installCNIManagerDaemonset(ctx, clientset, os.Getenv(envImageTag), logDir); err != nil { log.Print(err) return } @@ -87,8 +90,8 @@ func TestMain(m *testing.M) { if installopt := os.Getenv(envInstallCNS); installopt != "" { // create dirty cns ds - if installCNS, err := strconv.ParseBool(installopt); err != nil && installCNS == true { - if cnscleanup, err = installCNSDaemonset(ctx, clientset, os.Getenv(envImageTag)); err != nil { + if installCNS, err := strconv.ParseBool(installopt); err == nil && installCNS == true { + if cnscleanup, err = installCNSDaemonset(ctx, clientset, os.Getenv(envImageTag), logDir); err != nil { return } } @@ -99,7 +102,7 @@ func TestMain(m *testing.M) { exitCode = m.Run() } -func installCNSDaemonset(ctx context.Context, clientset *kubernetes.Clientset, imageTag string) (func() error, error) { +func installCNSDaemonset(ctx context.Context, clientset *kubernetes.Clientset, imageTag, logDir string) (func() error, error) { var ( err error cns v1.DaemonSet @@ -119,7 +122,6 @@ func installCNSDaemonset(ctx context.Context, clientset *kubernetes.Clientset, i cnsDaemonsetClient := clientset.AppsV1().DaemonSets(cns.Namespace) log.Printf("Installing CNS with image %s", cns.Spec.Template.Spec.Containers[0].Image) - // setup the CNS configmap if err := mustSetupConfigMap(ctx, clientset, cnsConfigMapPath); err != nil { return nil, err @@ -144,6 +146,9 @@ func installCNSDaemonset(ctx context.Context, clientset *kubernetes.Clientset, i } cleanupds := func() error { + if err := exportLogsByLabelSelector(ctx, clientset, cns.Namespace, cnsLabelSelector, logDir); err != nil { + return err + } if err := mustDeleteDaemonset(ctx, cnsDaemonsetClient, cns); err != nil { return err } @@ -153,7 +158,7 @@ func installCNSDaemonset(ctx context.Context, clientset *kubernetes.Clientset, i return cleanupds, nil } -func installCNIManagerDaemonset(ctx context.Context, clientset *kubernetes.Clientset, imageTag string) (func() error, error) { +func installCNIManagerDaemonset(ctx context.Context, clientset *kubernetes.Clientset, imageTag, logDir string) (func() error, error) { var ( err error cni v1.DaemonSet @@ -179,6 +184,9 @@ func installCNIManagerDaemonset(ctx context.Context, clientset *kubernetes.Clien } cleanupds := func() error { + if err := exportLogsByLabelSelector(ctx, clientset, cni.Namespace, cniLabelSelector, logDir); err != nil { + return err + } if err := mustDeleteDaemonset(ctx, cniDaemonsetClient, cni); err != nil { return err } diff --git a/test/integration/utils_test.go b/test/integration/utils_test.go index b928dd0502..c6a33505c2 100644 --- a/test/integration/utils_test.go +++ b/test/integration/utils_test.go @@ -3,8 +3,10 @@ package k8s import ( + "bytes" "context" "errors" + "io" "log" "strings" "time" @@ -220,6 +222,60 @@ func waitForPodsRunning(ctx context.Context, clientset *kubernetes.Clientset, na return nil } - retrier := retry.Retrier{Attempts: 10, Delay: 2 * time.Second} + retrier := retry.Retrier{Attempts: 10, Delay: 6 * time.Second} return retrier.Do(ctx, checkPodIPsFn) } + + +func exportLogsByLabelSelector(ctx context.Context, clientset *kubernetes.Clientset, namespace, labelselector, logDir string) error { + podsClient := clientset.CoreV1().Pods(namespace) + podLogOpts := corev1.PodLogOptions{} + logExtension := ".log" + podList, err := podsClient.List(ctx, metav1.ListOptions{LabelSelector: labelselector}) + if err != nil { + return err + } + + for _, pod := range podList.Items { + req := podsClient.GetLogs(pod.Name, &podLogOpts) + podLogs, err := req.Stream(ctx) + if err != nil { + return err + } + defer podLogs.Close() + + buf := new(bytes.Buffer) + _, err = io.Copy(buf, podLogs) + if err != nil { + return err + } + str := buf.String() + err = writeToFile(logDir, pod.Name + logExtension, str) + if err != nil { + return err + } + } + return nil +} + +func writeToFile(dir, fileName, str string) error { + if _, err := os.Stat(dir); os.IsNotExist(err) { + // your dir does not exist + os.MkdirAll(dir, 0666) + } + // open output file + f, err := os.Create(dir + fileName) + if err != nil { + return err + } + // close fo on exit and check for its returned error + defer func() { + if err := f.Close(); err != nil { + panic(err) + } + }() + + // If write went ok then err is nil + _, err = f.WriteString(str) + return err +} diff --git a/tools/acncli/Dockerfile b/tools/acncli/Dockerfile index 0adbe4c315..c18523403d 100644 --- a/tools/acncli/Dockerfile +++ b/tools/acncli/Dockerfile @@ -19,4 +19,4 @@ ENV AZURE_CNI_TENANCY=singletenancy ENV AZURE_CNI_IPAM=azure-cns ENV AZURE_CNI_MODE=transparent -CMD ["./acn", "manager", "-f"] +ENTRYPOINT ["./acn", "cni", "manager", "--follow", "--mode", "transparent"] diff --git a/tools/acncli/api/constants.go b/tools/acncli/api/constants.go index 7c535a1a7f..18a7e64fe7 100644 --- a/tools/acncli/api/constants.go +++ b/tools/acncli/api/constants.go @@ -82,7 +82,7 @@ var ( FlagTenancy: Singletenancy, FlagIPAM: AzureVNETIPAM, FlagExempt: AzureTelemetryBin + "," + AzureTelemetryConfig, - FlagMode: Bridge, + FlagMode: Transparent, FlagTarget: Local, FlagBinDirectory: DefaultBinDirLinux, FlagConflistDirectory: DefaultConflistDirLinux,