From 9d4c59df5c6ca2ea01ce81963b2ba30d6eaca76f Mon Sep 17 00:00:00 2001 From: Sebastian Stauch Date: Wed, 5 Apr 2023 14:53:13 +0200 Subject: [PATCH] extend e2e tests --- test/e2e/network_test.go | 110 ++++++++++++++++++++- test/templates/network-test.yaml.tpl | 137 +++++++++++++++++++++++++++ test/templates/templates.go | 12 +++ 3 files changed, 258 insertions(+), 1 deletion(-) create mode 100644 test/templates/network-test.yaml.tpl create mode 100644 test/templates/templates.go diff --git a/test/e2e/network_test.go b/test/e2e/network_test.go index 05c09d81d..425199b67 100644 --- a/test/e2e/network_test.go +++ b/test/e2e/network_test.go @@ -6,26 +6,134 @@ package e2e_test import ( "context" + "path/filepath" "time" + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + + "github.com/gardener/gardener-extension-networking-calico/test/templates" + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + "github.com/gardener/gardener/pkg/client/kubernetes" + gardenerutils "github.com/gardener/gardener/pkg/utils/gardener" + kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes" + "github.com/gardener/gardener/test/framework" + "github.com/gardener/gardener/test/utils/shoots/access" . "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" . "github.com/onsi/gomega" + "sigs.k8s.io/controller-runtime/pkg/client" ) var _ = Describe("Network Extension Tests", Label("Network"), func() { f := defaultShootCreationFramework() f.Shoot = defaultShoot("e2e-default") - It("Create Shoot, Delete Shoot", Label("good-case"), func() { + It("Create Shoot, Test Network, Delete Shoot", Label("good-case"), func() { By("Create Shoot") ctx, cancel := context.WithTimeout(parentCtx, 15*time.Minute) defer cancel() Expect(f.CreateShootAndWaitForCreation(ctx, false)).To(Succeed()) f.Verify() + By("Test Network") + ctx, cancel = context.WithTimeout(parentCtx, 15*time.Minute) + defer cancel() + values := struct { + HelmDeployNamespace string + KubeVersion string + }{ + templates.NetworkTestNamespace, + f.Shoot.Spec.Kubernetes.Version, + } + + var err error + f.GardenClient, err = kubernetes.NewClientFromFile("", f.ShootFramework.Config.GardenerConfig.GardenerKubeconfig, + kubernetes.WithClientOptions(client.Options{Scheme: kubernetes.GardenScheme}), + kubernetes.WithAllowedUserFields([]string{kubernetes.AuthTokenFile}), + kubernetes.WithDisabledCachedClient(), + ) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + shootKubeconfigSecret := &corev1.Secret{} + gardenClient := f.GardenClient.Client() + gardenClient.Get(ctx, kubernetesutils.Key(f.Shoot.Namespace, gardenerutils.ComputeShootProjectSecretName(f.Shoot.Name, gardenerutils.ShootProjectSecretSuffixKubeconfig)), shootKubeconfigSecret) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + f.ShootFramework.ShootClient, err = access.CreateShootClientFromAdminKubeconfig(ctx, f.GardenClient, f.Shoot) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + newShootKubeconfigSecret := &corev1.Secret{ObjectMeta: v1.ObjectMeta{ + Name: "kubeconfig", + Namespace: values.HelmDeployNamespace}, + Data: map[string][]byte{"kubeconfig": shootKubeconfigSecret.Data["kubeconfig"]}, + } + err = f.ShootFramework.ShootClient.Client().Create(ctx, newShootKubeconfigSecret) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + resourceDir, err := filepath.Abs(filepath.Join("..")) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + f.TemplatesDir = filepath.Join(resourceDir, "templates") + + err = f.RenderAndDeployTemplate(ctx, f.ShootFramework.ShootClient, templates.NetworkTestName, values) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + time.Sleep(30 * time.Second) + err = f.ShootFramework.WaitUntilDaemonSetIsRunning( + ctx, + f.ShootFramework.ShootClient.Client(), + "host-network", + values.HelmDeployNamespace, + ) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + err = f.ShootFramework.WaitUntilDaemonSetIsRunning( + ctx, + f.ShootFramework.ShootClient.Client(), + "pod-network", + values.HelmDeployNamespace, + ) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + By("Network-test daemonsets were deployed successfully!") + + By("Check if network-test fails or succeeds!") + + // wait one minute until results are collected + time.Sleep(60 * time.Second) + + podListHostNetwork, err := framework.GetPodsByLabels(ctx, labels.SelectorFromSet(map[string]string{ + v1beta1constants.LabelApp: "host-network", + }), f.ShootFramework.ShootClient, values.HelmDeployNamespace) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + podListPodNetwork, err := framework.GetPodsByLabels(ctx, labels.SelectorFromSet(map[string]string{ + v1beta1constants.LabelApp: "pod-network", + }), f.ShootFramework.ShootClient, values.HelmDeployNamespace) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + containerStatuses := []corev1.ContainerStatus{} + for _, pod := range podListHostNetwork.Items { + containerStatuses = append(containerStatuses, pod.Status.ContainerStatuses...) + } + + for _, pod := range podListPodNetwork.Items { + containerStatuses = append(containerStatuses, pod.Status.ContainerStatuses...) + } + succeeded := true + for _, containerStatus := range containerStatuses { + if containerStatus.RestartCount > 0 { + succeeded = false + break + } + } + By("Delete Shoot") ctx, cancel = context.WithTimeout(parentCtx, 15*time.Minute) defer cancel() Expect(f.DeleteShootAndWaitForDeletion(ctx, f.Shoot)).To(Succeed()) + + By("Network Test status") + gomega.Expect(succeeded).To(BeTrue()) }) }) diff --git a/test/templates/network-test.yaml.tpl b/test/templates/network-test.yaml.tpl new file mode 100644 index 000000000..02bb0b335 --- /dev/null +++ b/test/templates/network-test.yaml.tpl @@ -0,0 +1,137 @@ +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: host-network + namespace: {{ .HelmDeployNamespace }} + labels: + app: host-network +spec: +spec: + selector: + matchLabels: + app: host-network + template: + metadata: + labels: + app: host-network + spec: + hostNetwork: true + containers: + - name: host-network-test-connectivity + image: "ubuntu" + command: + - /bin/bash + - -c + - | + apt-get update && apt-get install -y curl iputils-ping; /script/network-test.sh + volumeMounts: + - name: networking-test + mountPath: /script + volumes: + - name: networking-test + configMap: + defaultMode: 511 + name: network-test +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: pod-network + namespace: {{ .HelmDeployNamespace }} + labels: + app: pod-network +spec: + selector: + matchLabels: + app: pod-network + template: + metadata: + labels: + app: pod-network + spec: + containers: + - name: pod-network-test-connectivity + image: "ubuntu" + command: + - /bin/bash + - -c + - | + apt-get update && apt-get install -y curl iputils-ping; /script/network-test.sh + volumeMounts: + - name: networking-test + mountPath: /script + volumes: + - name: networking-test + configMap: + defaultMode: 511 + name: network-test +--- +kind: Service +apiVersion: v1 +metadata: + name: host-network + namespace: {{ .HelmDeployNamespace }} +spec: + selector: + app: host-network + ports: + port: 80 +--- +kind: Service +apiVersion: v1 +metadata: + name: pod-network + namespace: {{ .HelmDeployNamespace }} +spec: + selector: + app: pod-network + ports: + port: 80 +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: network-test-rbac +subjects: + - kind: ServiceAccount + name: default + namespace: default +roleRef: + kind: ClusterRole + name: cluster-admin + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: network-test + namespace: {{ .HelmDeployNamespace }} +data: + network-test.sh: | + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + chmod +x ./kubectl + mv ./kubectl /usr/local/bin/kubectl + + function ping_ips() { + local service_name=$1 + echo "Testing $service_name" + ips=$(kubectl get endpoints "$service_name" -o jsonpath='{.subsets[*].addresses[*].ip}') + for ip in $ips; do + ip=$(echo "$ip" | tr -d '"') + ping -c 1 "$ip" > /dev/null + if [ $? -ne 0 ] + then + echo "ERROR: ping $ip failed" + exit 1 + fi + echo "$(date): $ip - succeeded" + done + } + + while true + do + sleep 15 + ping_ips "host-network" + ping_ips "pod-network" + done diff --git a/test/templates/templates.go b/test/templates/templates.go new file mode 100644 index 000000000..e50de5b74 --- /dev/null +++ b/test/templates/templates.go @@ -0,0 +1,12 @@ +// SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package templates + +const ( + + // NetworkTestName is the name of a network ping test for calico + NetworkTestName = "network-test.yaml.tpl" + NetworkTestNamespace = "default" +)