Bring Goldpinger tests to ACN

.pipelines/pipeline.yaml

@@ -120,8 +120,6 @@ stages:
               bash <(curl -s https://codecov.io/bash)
               gocov convert coverage.out > coverage.json
               gocov-xml < coverage.json > coverage.xml
-              echo listing cluster definitions
-              ls $(modulePath)/test/e2e/kubernetes/*
             workingDirectory: "$(modulePath)"
             name: "Coverage"
             displayName: "Generate Coverage Reports"
@@ -181,7 +179,7 @@ stages:
               pathtoPublish: "$(Build.ArtifactStagingDirectory)"
             condition: succeeded()
 
-          - publish: $(modulePath)/test/e2e/kubernetes/
+          - publish: $(modulePath)/test/apimodels/
             artifact: clusterdefinitions
 
           - task: AzureCLI@1

go.sum

@@ -85,6 +85,7 @@ github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docker/libnetwork v0.5.6 h1:hnGiypBsZR6PW1I8lqaBHh06U6LCJbI3IhOvfsZiymY=
 github.com/docker/libnetwork v0.5.6/go.mod h1:93m0aTqz6z+g32wla4l4WxTrdtvBRmVzYRkYvasA5Z8=
+github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96 h1:cenwrSVm+Z7QLSV/BsnenAOcDXdX4cMv4wP0B/5QbPg=
 github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=

test/integration/goldpinger/client.go

@@ -0,0 +1,36 @@
+// +build integration
+
+package goldpinger
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+)
+
+type Client struct {
+	Host string
+}
+
+func (c *Client) CheckAll(ctx context.Context) (CheckAllJSON, error) {
+	endpoint := fmt.Sprintf("%s/check_all", c.Host)
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
+	if err != nil {
+		return CheckAllJSON{}, err
+	}
+
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return CheckAllJSON{}, err
+	}
+	defer res.Body.Close()
+
+	var jsonResp CheckAllJSON
+	if err := json.NewDecoder(res.Body).Decode(&jsonResp); err != nil {
+		return CheckAllJSON{}, err
+	}
+
+	return jsonResp, nil
+}

test/integration/goldpinger/models.go

@@ -0,0 +1,38 @@
+// +build integration
+
+package goldpinger
+
+import "time"
+
+type CheckAllJSON struct {
+	DNSResults map[string]interface{}   `json:"dnsResults"`
+	Hosts      []HostJSON               `json:"hosts"`
+	Responses  map[string]PodStatusJSON `json:"responses"`
+}
+
+type HostJSON struct {
+	HostIP  string `json:"hostIP"`
+	PodIP   string `json:"podIP"`
+	PodName string `json:"podName"`
+}
+
+type PodStatusJSON struct {
+	HostIP   string       `json:"HostIP"`
+	OK       bool         `json:"OK"`
+	PodIP    string       `json:"PodIP"`
+	Response ResponseJSON `json:"response"`
+}
+
+type ResponseJSON struct {
+	DNSResults map[string]interface{}    `json:"dnsResults"`
+	PodResults map[string]PingResultJSON `json:"podResults"`
+}
+
+type PingResultJSON struct {
+	HostIP                   string    `json:"HostIP"`
+	OK                       bool      `json:"OK"`
+	PodIP                    string    `json:"PodIP"`
+	PingTime                 time.Time `json:"PingTime"`
+	ResponseTimeMilliseconds int       `json:"response-time-ms"`
+	StatusCode               int       `json:"status-code"`
+}

test/integration/goldpinger/utils.go

@@ -0,0 +1,71 @@
+// +build integration
+
+package goldpinger
+
+import "fmt"
+
+type ClusterStats CheckAllJSON
+
+func (c ClusterStats) AllPodsHealthy() bool {
+	if len(c.Responses) == 0 {
+		return false
+	}
+
+	for _, podStatus := range c.Responses {
+		if !podStatus.OK {
+			return false
+		}
+	}
+	return true
+}
+
+func (c ClusterStats) AllPingsHealthy() bool {
+	if !c.AllPodsHealthy() {
+		return false
+	}
+	for _, podStatus := range c.Responses {
+		for _, pingResult := range podStatus.Response.PodResults {
+			if !pingResult.OK {
+				return false
+			}
+		}
+	}
+	return true
+}
+
+type stringSet map[string]struct{}
+
+func (set stringSet) add(s string) {
+	set[s] = struct{}{}
+}
+
+func (c ClusterStats) PrintStats() {
+	podCount := len(c.Hosts)
+	nodes := make(stringSet)
+	healthyPods := make(stringSet)
+	pingCount := 0
+	healthyPingCount := 0
+
+	for _, podStatus := range c.Responses {
+		nodes.add(podStatus.HostIP)
+		if podStatus.OK {
+			healthyPods.add(podStatus.PodIP)
+		}
+		for _, pingStatus := range podStatus.Response.PodResults {
+			pingCount++
+			if pingStatus.OK {
+				healthyPingCount++
+			}
+		}
+	}
+
+	format := "cluster stats - " +
+		"nodes in use: %d, " +
+		"pod count: %d, " +
+		"pod health percentage: %2.2f, " +
+		"ping health percentage: %2.2f\n"
+
+	podHealthPct := (float64(len(healthyPods)) / float64(podCount)) * 100
+	pingHealthPct := (float64(healthyPingCount) / float64(pingCount)) * 100
+	fmt.Printf(format, len(nodes), podCount, podHealthPct, pingHealthPct)
+}

test/integration/k8s_test.go

@@ -0,0 +1,213 @@
+// +build integration
+
+package k8s
+
+import (
+	"context"
+	//"dnc/test/integration/goldpinger"
+	"errors"
+	"flag"
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/Azure/azure-container-networking/test/integration/goldpinger"
+	"github.com/Azure/azure-container-networking/test/integration/retry"
+	v1 "k8s.io/client-go/kubernetes/typed/apps/v1"
+
+	apiv1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/util/homedir"
+)
+
+var (
+	defaultRetrier      = retry.Retrier{Attempts: 15, Delay: time.Second}
+	kubeconfig          = flag.String("test-kubeconfig", filepath.Join(homedir.HomeDir(), ".kube", "config"), "(optional) absolute path to the kubeconfig file")
+	delegatedSubnetID   = flag.String("delegated-subnet-id", "", "delegated subnet id for node labeling")
+	delegatedSubnetName = flag.String("subnet-name", "", "subnet name for node labeling")
+)
+
+const (
+	subnetIDNodeLabelEnvVar   = "DELEGATED_SUBNET_ID_NODE_LABEL"
+	subnetNameNodeLabelEnvVar = "SUBNET_NAME_NODE_LABEL"
+)
+
+func shouldLabelNodes() bool {
+	if *delegatedSubnetID == "" {
+		*delegatedSubnetID = os.Getenv(subnetIDNodeLabelEnvVar)
+	}
+	if *delegatedSubnetName == "" {
+		*delegatedSubnetName = os.Getenv(subnetNameNodeLabelEnvVar)
+	}
+	return *delegatedSubnetID != "" && *delegatedSubnetName != ""
+}
+
+/*
+
+In order to run the tests below, you need a k8s cluster and its kubeconfig.
+If no kubeconfig is passed, the test will attempt to find one in the default location for kubectl config.
+The test will also attempt to label the nodes if the appropriate flags or environment variables are set.
+Run the tests as follows:
+
+go test -v . [-args test-kubeconfig=<...> delegated-subnet-id=<...> subnet-name=<...>]
+
+todo: consider adding the following scenarios
+- [x] All pods should be assigned an IP.
+- [ ] All pod IPs should belong to the delegated subnet and not overlap host subnet.
+- [x] All pods should be able to ping each other.
+- [ ] All pods should be able to ping nodes. Daemonset with hostnetworking?
+- [ ] All pods should be able to reach public internet. Enable hosts to ping in goldpinger deployment.
+
+- [x] All scenarios above should be valid during deployment scale up
+- [x] All scenarios above should be valid during deployment scale down
+
+- [ ] All scenarios above should be valid during node scale up (i.e more nodes == more NNCs)
+- [ ] All scenarios above should be valid during node scale down
+
+todo:
+  - Need hook for `az aks scale --g <resource group> --n <cluster name> --node-count <prev++> --nodepool-name <np name>`
+  - Need hook for pubsub client to verify that no secondary CAs are leaked
+  - Check CNS ipam pool?
+  - Check NNC in apiserver?
+*/
+
+func TestPodScaling(t *testing.T) {
+	clientset := mustGetClientset(t)
+	restConfig := mustGetRestConfig(t)
+	deployment := mustParseDeployment(t, "testdata/goldpinger/deployment.yaml")
+
+	ctx := context.Background()
+
+	if shouldLabelNodes() {
+		mustLabelSwiftNodes(t, ctx, clientset, *delegatedSubnetID, *delegatedSubnetName)
+	} else {
+		t.Log("swift node labels not passed or set. skipping labeling")
+	}
+
+	rbacCleanUpFn := mustSetUpRBAC(t, ctx, clientset)
+	deploymentsClient := clientset.AppsV1().Deployments(deployment.Namespace)
+	mustCreateDeployment(t, ctx, deploymentsClient, deployment)
+
+	t.Cleanup(func() {
+		t.Log("cleaning up resources")
+		rbacCleanUpFn(t)
+
+		if err := deploymentsClient.Delete(ctx, deployment.Name, metav1.DeleteOptions{}); err != nil {
+			t.Log(err)
+		}
+	})
+
+	counts := []int{10, 20, 50, 10}
+
+	for _, c := range counts {
+		count := c
+		t.Run(fmt.Sprintf("replica count %d", count), func(t *testing.T) {
+			replicaCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
+			defer cancel()
+
+			if err := updateReplicaCount(t, replicaCtx, deploymentsClient, deployment.Name, count); err != nil {
+				t.Fatalf("could not scale deployment: %v", err)
+			}
+
+			t.Run("all pods have IPs assigned", func(t *testing.T) {
+				podsClient := clientset.CoreV1().Pods(deployment.Namespace)
+
+				checkPodIPsFn := func() error {
+					podList, err := podsClient.List(ctx, metav1.ListOptions{LabelSelector: "app=goldpinger"})
+					if err != nil {
+						return err
+					}
+
+					if len(podList.Items) == 0 {
+						return errors.New("no pods scheduled")
+					}
+
+					for _, pod := range podList.Items {
+						if pod.Status.Phase == apiv1.PodPending {
+							return errors.New("some pods still pending")
+						}
+					}
+
+					for _, pod := range podList.Items {
+						if pod.Status.PodIP == "" {
+							return errors.New("a pod has not been allocated an IP")
+						}
+					}
+
+					return nil
+				}
+				err := defaultRetrier.Do(ctx, checkPodIPsFn)
+				if err != nil {
+					t.Fatalf("not all pods were allocated IPs: %v", err)
+				}
+				t.Log("all pods have been allocated IPs")
+			})
+
+			t.Run("all pods can ping each other", func(t *testing.T) {
+				pf, err := NewPortForwarder(restConfig)
+				if err != nil {
+					t.Fatal(err)
+				}
+
+				portForwardCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
+				defer cancel()
+
+				var streamHandle PortForwardStreamHandle
+				portForwardFn := func() error {
+					t.Log("attempting port forward")
+					handle, err := pf.Forward(ctx, "default", "app=goldpinger", 8080, 8080)
+					if err != nil {
+						return err
+					}
+					streamHandle = handle
+					return nil
+				}
+				if err := defaultRetrier.Do(portForwardCtx, portForwardFn); err != nil {
+					t.Fatalf("could not start port forward within 30s: %v", err)
+				}
+				defer streamHandle.Stop()
+
+				gpClient := goldpinger.Client{Host: streamHandle.Url()}
+
+				clusterCheckCtx, cancel := context.WithTimeout(ctx, time.Minute)
+				defer cancel()
+				clusterCheckFn := func() error {
+					clusterState, err := gpClient.CheckAll(clusterCheckCtx)
+					if err != nil {
+						return err
+					}
+
+					stats := goldpinger.ClusterStats(clusterState)
+					stats.PrintStats()
+					if stats.AllPingsHealthy() {
+						return nil
+					}
+
+					return errors.New("not all pings are healthy")
+				}
+				retrier := retry.Retrier{Attempts: 10, Delay: 5 * time.Second}
+				if err := retrier.Do(clusterCheckCtx, clusterCheckFn); err != nil {
+					t.Fatalf("cluster could not reach healthy state: %v", err)
+				}
+
+				t.Log("all pings successful!")
+			})
+		})
+	}
+}
+
+func updateReplicaCount(t *testing.T, ctx context.Context, deployments v1.DeploymentInterface, name string, replicas int) error {
+	return defaultRetrier.Do(ctx, func() error {
+		res, err := deployments.Get(ctx, name, metav1.GetOptions{})
+		if err != nil {
+			return err
+		}
+
+		t.Logf("setting deployment %s to %d replicas", name, replicas)
+		res.Spec.Replicas = int32ptr(int32(replicas))
+		_, err = deployments.Update(ctx, res, metav1.UpdateOptions{})
+		return err
+	})
+}