Integration Testing Foundations

go.mod

@@ -4,43 +4,45 @@ go 1.15
 
 require (
 	github.com/datastax/cass-operator v1.5.1-0.20210112050706-a45fd80b40e0
-	github.com/gruntwork-io/terratest v0.30.15
+	github.com/gruntwork-io/terratest v0.32.4
 	github.com/k8ssandra/reaper-operator v0.0.0-20210122200305-0d6525659e9d
 	github.com/onsi/ginkgo v1.14.2
 	github.com/onsi/gomega v1.10.3
-	github.com/stretchr/testify v1.6.1 // indirect
+	github.com/stretchr/testify v1.6.1
 	github.com/traefik/traefik/v2 v2.3.7
-	k8s.io/api v0.18.6
-	k8s.io/apimachinery v0.18.6
+	github.com/trivago/cluecumber-report-plugin v0.0.0-20210302121305-b2e68b2c78dd // indirect
+	k8s.io/api v0.19.3
+	k8s.io/apimachinery v0.19.3
 	k8s.io/client-go v12.0.0+incompatible
 	k8s.io/helm v2.16.3+incompatible
 	sigs.k8s.io/controller-runtime v0.6.4
+
 )
 
 replace (
-    github.com/docker/docker => github.com/docker/engine v1.4.2-0.20200204220554-5f6d6f3f2203
 	github.com/abbot/go-http-auth => github.com/containous/go-http-auth v0.4.1-0.20200324110947-a37a7636d23e
+	github.com/docker/docker => github.com/docker/engine v1.4.2-0.20200204220554-5f6d6f3f2203
 	github.com/go-check/check => github.com/containous/check v0.0.0-20170915194414-ca0bf163426a
-	k8s.io/api => k8s.io/api v0.18.6
-	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.18.6
-	k8s.io/apimachinery => k8s.io/apimachinery v0.18.6
-	k8s.io/apiserver => k8s.io/apiserver v0.18.6
-	k8s.io/cli-runtime => k8s.io/cli-runtime v0.18.6
-	k8s.io/client-go => k8s.io/client-go v0.18.6
-	k8s.io/cloud-provider => k8s.io/cloud-provider v0.18.6
-	k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.18.6
-	k8s.io/code-generator => k8s.io/code-generator v0.18.6
-	k8s.io/component-base => k8s.io/component-base v0.18.6
-	k8s.io/cri-api => k8s.io/cri-api v0.18.6
-	k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.18.6
-	k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.18.6
-	k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.18.6
-	k8s.io/kube-proxy => k8s.io/kube-proxy v0.18.6
-	k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.18.6
-	k8s.io/kubectl => k8s.io/kubectl v0.18.6
-	k8s.io/kubelet => k8s.io/kubelet v0.18.6
-	k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.18.6
-	k8s.io/metrics => k8s.io/metrics v0.18.6
-	k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.18.6
+	k8s.io/api => k8s.io/api v0.19.3
+	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.19.3
+	k8s.io/apimachinery => k8s.io/apimachinery v0.19.3
+	k8s.io/apiserver => k8s.io/apiserver v0.19.3
+	k8s.io/cli-runtime => k8s.io/cli-runtime v0.19.3
+	k8s.io/client-go => k8s.io/client-go v0.19.3
+	k8s.io/cloud-provider => k8s.io/cloud-provider v0.19.3
+	k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.19.3
+	k8s.io/code-generator => k8s.io/code-generator v0.19.3
+	k8s.io/component-base => k8s.io/component-base v0.19.3
+	k8s.io/cri-api => k8s.io/cri-api v0.19.3
+	k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.19.3
+	k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.19.3
+	k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.19.3
+	k8s.io/kube-proxy => k8s.io/kube-proxy v0.19.3
+	k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.19.3
+	k8s.io/kubectl => k8s.io/kubectl v0.19.3
+	k8s.io/kubelet => k8s.io/kubelet v0.19.3
+	k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.19.3
+	k8s.io/metrics => k8s.io/metrics v0.19.3
+	k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.19.3
 	sigs.k8s.io/controller-runtime => sigs.k8s.io/controller-runtime v0.6.4
 )

tests/integration/kubectl/kubectl_integration_test.go

@@ -0,0 +1,37 @@
+package kubectl
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/gruntwork-io/terratest/modules/k8s"
+	"github.com/gruntwork-io/terratest/modules/random"
+	"github.com/stretchr/testify/require"
+	authv1 "k8s.io/api/authorization/v1"
+)
+
+// An example of verifying the kubectl is operational and that
+// authorization for basic operations.
+func TestKubectlAuthorization(t *testing.T) {
+
+	t.Parallel()
+
+	ns := fmt.Sprintf("test-ns-%s", strings.ToLower(random.UniqueId()))
+	kubectlOptions := k8s.NewKubectlOptions("", "", ns)
+
+	adminGetServiceAction := authv1.ResourceAttributes{
+		Namespace: ns,
+		Verb:      "get",
+		Resource:  "service",
+	}
+
+	adminGetPodAction := authv1.ResourceAttributes{
+		Namespace: ns,
+		Verb:      "get",
+		Resource:  "pod",
+	}
+
+	require.True(t, k8s.CanIDo(t, kubectlOptions, adminGetServiceAction))
+	require.True(t, k8s.CanIDo(t, kubectlOptions, adminGetPodAction))
+}

tests/integration/operator/operator_integration_test.go

@@ -0,0 +1,114 @@
+package operator
+
+import (
+	"github.com/gruntwork-io/terratest/modules/k8s"
+	. "github.com/k8ssandra/k8ssandra/tests/integration/util"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	_ "k8s.io/api/apps/v1"
+	_ "k8s.io/api/core/v1"
+	_ "k8s.io/apimachinery/pkg/api/errors"
+	_ "k8s.io/apimachinery/pkg/apis/meta/v1"
+	_ "k8s.io/apimachinery/pkg/types"
+	"testing"
+	"time"
+)
+
+const (
+	SuiteName            = "k8ssandra-operator-integration-suite"
+	KindClusterName      = "k8ssandra-cluster"
+	K8ssandraContext     = "kind-k8ssandra-cluster"
+	K8ssandraReleaseName = "k8ssandra"
+	NetworkReleaseName   = "traefik"
+	Namespace            = "k8ssandra"
+
+	K8ssandraSingeNodeValues = "../preconditions/k8ssandra-single-node.yaml"
+	TraefikValues            = "../preconditions/k8ssandra-traefik-values.yaml"
+	KubeConfigFile           = "../preconditions/k8ssandra-config-test.yaml"
+	KindConfigFile           = "../preconditions/k8ssandra-kind-config.yaml"
+	KindImage                = "kindest/node:v1.20.2"
+
+	includeDatacenterVerification = false
+)
+
+func Test(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, SuiteName)
+}
+
+// setup used for creating test option context and cleanup
+func setup() K8ssandraOptions {
+
+	options := CreateK8ssandraOptions(k8s.NewKubectlOptions(K8ssandraContext,
+		"", Namespace), K8ssandraReleaseName, NetworkReleaseName)
+	ConfigureContext(options.GetOperatorCtx(), KubeConfigFile)
+
+	// Check if kind is available.
+	if IsKindClusterExisting(KindClusterName) {
+		DeleteKindCluster(KindClusterName)
+	}
+
+	// Create Cluster fresh
+	CreateKindCluster(KindClusterDetail{Name: KindClusterName,
+		Image: KindImage, ConfigFile: KindConfigFile})
+
+	RepoUpdate(options.GetOperatorCtx())
+	return options
+}
+
+var _ = Describe(SuiteName, func() {
+
+	Describe("k8ssandra Operator and Cassandra integration", func() {
+
+		var ko K8ssandraOptions
+		BeforeSuite(func() {
+			ko = setup()
+			Expect(ko).ToNot(BeNil())
+
+			By("Expecting Traefik to be deployed")
+			InstallChartWithValues(ko.GetNetworkCtx(), "traefik/traefik", TraefikValues)
+			Expect(IsReleaseDeployed(ko.GetNetworkCtx())).Should(BeTrue())
+			k8s.WaitUntilAllNodesReady(GinkgoT(), ko.GetNetworkCtx().KubeOptions, 15, 3*time.Second)
+		})
+
+		It("should install utilizing charts; k8ssandra operator then cluster", func() {
+
+			By("Expecting k8ssandra to be deployed.")
+			ctx := ko.GetOperatorCtx()
+			ctx.SetArgs([]string{
+				// "--set", "datacenters[0].size=2",
+				"--set", "k8ssandra.Namespace=" + ko.GetOperatorCtx().Namespace,
+				"--set", "clusterName=" + KindClusterName})
+
+			InstallChartWithValues(ctx, "k8ssandra/k8ssandra", K8ssandraSingeNodeValues)
+
+			WaitFor(func() bool { return IsReleaseDeployed(ko.GetOperatorCtx()) },
+				"k8ssandra to be deployed", 6, 30)
+			Expect(IsReleaseDeployed(ko.GetOperatorCtx())).Should(BeTrue())
+			k8s.WaitUntilAllNodesReady(GinkgoT(), ko.GetOperatorCtx().KubeOptions, 15, 3*time.Second)
+
+			By("Expecting a running k8ssandra cass-operator with pod available.")
+			var cassOperatorLabel = []PodLabel{{Key: "app.kubernetes.io/name", Value: "cass-operator"}}
+			WaitFor(func() bool { return IsPodWithLabel(ko.GetOperatorCtx(), cassOperatorLabel) },
+				"Cass-operator with pod availability", 10, 300)
+			Expect(IsPodWithLabel(ko.GetOperatorCtx(), cassOperatorLabel)).Should(BeTrue())
+
+			// Requires considerable time, optional verification as a post test can do this after some time.
+			if includeDatacenterVerification {
+
+				By("Expecting a datacenter with name dc1.")
+				Expect(IsExisting(ko.GetOperatorCtx(), "CassandraDatacenter",
+					"cassandradatacenter.cassandra.datastax.com/dc1")).Should(BeTrue())
+				WaitFor(func() bool { return IsDeploymentReady(ko.GetOperatorCtx(), "cass-operator") },
+					"Deployment of cass-operator", 10, 60)
+
+				By("Expecting datacenter with node state started.")
+				var labels = []PodLabel{{Key: "cassandra.datastax.com/datacenter", Value: "dc1"},
+					{Key: "cassandra.datastax.com/node-state", Value: "Started"}}
+				WaitFor(func() bool { return IsPodWithLabel(ko.GetOperatorCtx(), labels) },
+					"Cassdc dc1 Running", 15, 600)
+			}
+
+		})
+	})
+})

tests/integration/preconditions/k8ssandra-config-test.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Config
+preferences: {}
+current-context: "kind-k8ssandra-cluster"
+clusters:
+  - cluster:
+    name: "k8ssandra-cluster"
+contexts:
+  - context:
+      cluster: "k8ssandra-cluster"
+      namespace: "k8ssandra"
+      user: "k8ssandra-dev"
+    name: "kind-k8ssandra-cluster"
+users:
+  - name: "k8ssandra-dev"

tests/integration/preconditions/k8ssandra-kind-config.yaml

@@ -0,0 +1,35 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+  - role: worker
+  - role: worker
+  - role: worker
+  - role: control-plane
+    kubeadmConfigPatches:
+      - |
+        kind: InitConfiguration
+        nodeRegistration:
+          kubeletExtraArgs:
+            node-labels: "ingress-ready=true"
+    extraPortMappings:
+      - containerPort: 32080
+        hostPort: 8080
+        protocol: TCP
+      - containerPort: 32443
+        hostPort: 8443
+        protocol: TCP
+      - containerPort: 32090
+        hostPort: 9000
+        protocol: TCP
+      - containerPort: 30081
+        hostPort: 8081
+        protocol: TCP
+      - containerPort: 30082
+        hostPort: 8082
+        protocol: TCP
+      - containerPort: 32091
+        hostPort: 9042
+        protocol: TCP
+      - containerPort: 32092
+        hostPort: 9142
+        protocol: TCP

tests/integration/preconditions/k8ssandra-secret-config.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: medusa-bucket-key
+type: Opaque
+stringData:
+  # Note that this currently has to be set to medusa_s3_credentials!
+  medusa_s3_credentials: |-
+    [default]
+    aws_access_key_id = my_access_key
+    aws_secret_access_key = my_secret_key

tests/integration/preconditions/k8ssandra-setup.md

@@ -0,0 +1,38 @@
+
+## Integration Test Setup
+
+### Preconditions
+Ensure the following are installed in the integration test environment.
+- [Helm v3+](https://helm.sh/docs/intro/install/)
+- [Kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/)
+- [Kind](https://kind.sigs.k8s.io/)
+
+### Validate cluster connectivity
+`kubectl cluster-info`
+
+If cluster is not existing, install k8s cluster using Kind. 
+
+### Create Cluster
+`kind create cluster --name k8ssandra-cluster --image kindest/node:v1.18.2 --config ./k8ssandra-kind-config.yaml`
+
+`kind get clusters`
+
+### Validate
+Invoke the cluster-info again with the expected context used in the integration tests.
+
+`kubectl cluster-info --context kind-k8ssandra-cluster`
+
+### Troubleshoot
+To debug and diagnose any cluster problems, use:
+
+`kubectl cluster-info dump`
+
+Note: This will produces LOTS of details so piping output to file is recommended.
+
+### Running Tests
+It is recommended that adequate time is given for the integration tests to fully complete.  Subject to change, but recommended minimum of 5 minutes.
+
+Example:
+
+`go test -v -timeout=300s`
+

tests/integration/preconditions/k8ssandra-single-node.yaml

@@ -0,0 +1,31 @@
+cassandra:
+  version: "3.11.10"
+  cassandraLibDirVolume:
+    storageClass: standard
+    size: 5Gi
+  allowMultipleNodesPerWorker: true
+  heap:
+    size: 1G
+    newGenSize: 1G
+  resources:
+    requests:
+      cpu: 1000m
+      memory: 2Gi
+    limits:
+      cpu: 1000m
+      memory: 2Gi
+  datacenters:
+    - name: dc1
+      size: 1
+      racks:
+        - name: default
+kube-prometheus-stack:
+  grafana:
+  adminUser: admin
+  adminPassword: admin123
+stargate:
+  enabled: true
+  replicas: 1
+  heapMB: 256
+  cpuReqMillicores: 200
+  cpuLimMillicores: 1000