diff --git a/.github/workflows/composite-actions/kubernetes-e2e-tests/action.yaml b/.github/workflows/composite-actions/kubernetes-e2e-tests/action.yaml index 375f151efd1..c0bb2ec4c05 100644 --- a/.github/workflows/composite-actions/kubernetes-e2e-tests/action.yaml +++ b/.github/workflows/composite-actions/kubernetes-e2e-tests/action.yaml @@ -26,6 +26,7 @@ runs: env: GO_TEST_USER_ARGS: ${{ inputs.test-args }} -run "${{ inputs.run-regex }}" CLUSTER_NAME: ${{ inputs.cluster-name }} + CLUSTER_ID: ${{ inputs.cluster-id }} TEST_PKG: ./test/kubernetes/e2e/tests shell: bash run: make go-test diff --git a/.github/workflows/pr-kubernetes-tests.yaml b/.github/workflows/pr-kubernetes-tests.yaml index 459f838c6b7..630820a6780 100644 --- a/.github/workflows/pr-kubernetes-tests.yaml +++ b/.github/workflows/pr-kubernetes-tests.yaml @@ -117,9 +117,10 @@ jobs: cluster-name: kind test-args: ${{ matrix.test.go-test-args }} run-regex: ${{ matrix.test.go-test-run-regex }} + cluster-id: ${{ matrix.test.cluster-id }} - name: Archive bug report directory on failure if: ${{ failure() }} uses: actions/upload-artifact@v4 with: name: bug-report - path: ./_test/bug_report + path: ./_test/bug_report/${{ matrix.test.cluster-id }} diff --git a/Makefile b/Makefile index acb4e572d1f..77697a42f83 100644 --- a/Makefile +++ b/Makefile @@ -81,6 +81,11 @@ TEST_ASSET_DIR ?= $(ROOTDIR)/_test BUG_REPORT_DIR := $(TEST_ASSET_DIR)/bug_report $(BUG_REPORT_DIR): mkdir -p $(BUG_REPORT_DIR) + # Create bug report directory for each cluster in ci (cluster_one, cluster_two, etc.) + mkdir -p $(BUG_REPORT_DIR)/cluster_one + mkdir -p $(BUG_REPORT_DIR)/cluster_two + mkdir -p $(BUG_REPORT_DIR)/cluster_three + mkdir -p $(BUG_REPORT_DIR)/cluster_four # Used to install ca-certificates in GLOO_DISTROLESS_BASE_IMAGE PACKAGE_DONOR_IMAGE ?= debian:11 diff --git a/changelog/v1.17.0-beta28/add-istio-upstream-tests.yaml b/changelog/v1.17.0-beta28/add-istio-upstream-tests.yaml new file mode 100644 index 00000000000..d4a9e06c353 --- /dev/null +++ b/changelog/v1.17.0-beta28/add-istio-upstream-tests.yaml @@ -0,0 +1,7 @@ +changelog: + - type: NON_USER_FACING + issueLink: https://github.com/solo-io/solo-projects/issues/6048 + resolvesIssue: false + description: >- + Add Upstream resource e2e tests for Istio integration with disabled peer auth. Adds upgrade/downgrade tests + for Upstream sslConfig switching to Istio auto mtls. \ No newline at end of file diff --git a/docs/content/reference/api/github.com/solo-io/solo-apis/api/gloo-fed/fed/core/v1/placement.proto.sk.md b/docs/content/reference/api/github.com/solo-io/solo-apis/api/gloo-fed/fed/core/v1/placement.proto.sk.md index f2ffa91060e..198bf6bfd10 100644 --- a/docs/content/reference/api/github.com/solo-io/solo-apis/api/gloo-fed/fed/core/v1/placement.proto.sk.md +++ b/docs/content/reference/api/github.com/solo-io/solo-apis/api/gloo-fed/fed/core/v1/placement.proto.sk.md @@ -29,7 +29,8 @@ weight: 5 --- ### TemplateMetadata - + +Object Metadata to be written with the resource into the remote cluster ```yaml "annotations": map @@ -63,7 +64,7 @@ weight: 5 | Field | Type | Description | | ----- | ---- | ----------- | -| `clusters` | `map` | | +| `clusters` | `map` | map containing the name of the cluster, with the associated Cluster namespaces. | | `state` | [.core.fed.solo.io.PlacementStatus.State](../placement.proto.sk/#state) | | | `message` | `string` | | | `observedGeneration` | `int` | metadata.Generation of the resource which has been processed. | diff --git a/docs/content/reference/api/github.com/solo-io/solo-apis/api/gloo-fed/fed/v1/failover.proto.sk.md b/docs/content/reference/api/github.com/solo-io/solo-apis/api/gloo-fed/fed/v1/failover.proto.sk.md index e69a363617c..0b73d7e95e2 100644 --- a/docs/content/reference/api/github.com/solo-io/solo-apis/api/gloo-fed/fed/v1/failover.proto.sk.md +++ b/docs/content/reference/api/github.com/solo-io/solo-apis/api/gloo-fed/fed/v1/failover.proto.sk.md @@ -15,6 +15,7 @@ weight: 5 - [FailoverEndpoints](#failoverendpoints) - [LocalityLbTargets](#localitylbtargets) - [FailoverSchemeStatus](#failoverschemestatus) +- [Status](#status) - [State](#state) @@ -122,6 +123,31 @@ namespace: two +```yaml +"state": .fed.solo.io.FailoverSchemeStatus.State +"message": string +"observedGeneration": int +"processingTime": .google.protobuf.Timestamp +"namespacedStatuses": map + +``` + +| Field | Type | Description | +| ----- | ---- | ----------- | +| `state` | [.fed.solo.io.FailoverSchemeStatus.State](../failover.proto.sk/#state) | The current state of the resource. Deprecated: use namespacedStatuses instead. | +| `message` | `string` | A human readable message about the current state of the object. Deprecated: use namespacedStatuses instead. | +| `observedGeneration` | `int` | The most recently observed generation of the resource. This value corresponds to the `metadata.generation` of a kubernetes resource. Deprecated: use namespacedStatuses instead. | +| `processingTime` | [.google.protobuf.Timestamp](https://developers.google.com/protocol-buffers/docs/reference/csharp/class/google/protobuf/well-known-types/timestamp) | The time at which this status was recorded. Deprecated: use namespacedStatuses instead. | +| `namespacedStatuses` | `map` | Map of gloo fed controller namespace to FailoverScheme status. | + + + + +--- +### Status + + + ```yaml "state": .fed.solo.io.FailoverSchemeStatus.State "message": string diff --git a/docs/content/reference/api/github.com/solo-io/solo-apis/api/gloo-fed/fed/v1/instance.proto.sk.md b/docs/content/reference/api/github.com/solo-io/solo-apis/api/gloo-fed/fed/v1/instance.proto.sk.md index 6da38b53986..394a42731c4 100644 --- a/docs/content/reference/api/github.com/solo-io/solo-apis/api/gloo-fed/fed/v1/instance.proto.sk.md +++ b/docs/content/reference/api/github.com/solo-io/solo-apis/api/gloo-fed/fed/v1/instance.proto.sk.md @@ -92,6 +92,7 @@ weight: 5 "availableReplicas": int "readyReplicas": int "wasmEnabled": bool +"readConfigMulticlusterEnabled": bool "version": string "name": string "namespace": string @@ -107,6 +108,7 @@ weight: 5 | `availableReplicas` | `int` | The number of available proxy replicas. | | `readyReplicas` | `int` | The number of ready proxy replicas. | | `wasmEnabled` | `bool` | Whether or not this proxy supports wasm plugins. | +| `readConfigMulticlusterEnabled` | `bool` | Whether or not this proxy allows its config dumps to be read. | | `version` | `string` | The version. | | `name` | `string` | Name of the proxy instance, this is important to distinguish it from other proxy instances, which may be in the same namespace. | | `namespace` | `string` | Namespace in which the proxy is located. | @@ -205,6 +207,9 @@ Check describes the status of the objects powering the GlooInstance. "upstreams": .fed.solo.io.GlooInstanceSpec.Check.Summary "upstreamGroups": .fed.solo.io.GlooInstanceSpec.Check.Summary "proxies": .fed.solo.io.GlooInstanceSpec.Check.Summary +"rateLimitConfigs": .fed.solo.io.GlooInstanceSpec.Check.Summary +"matchableHttpGateways": .fed.solo.io.GlooInstanceSpec.Check.Summary +"matchableTcpGateways": .fed.solo.io.GlooInstanceSpec.Check.Summary "deployments": .fed.solo.io.GlooInstanceSpec.Check.Summary "pods": .fed.solo.io.GlooInstanceSpec.Check.Summary @@ -220,6 +225,9 @@ Check describes the status of the objects powering the GlooInstance. | `upstreams` | [.fed.solo.io.GlooInstanceSpec.Check.Summary](../instance.proto.sk/#summary) | Upstreams describes the Upstreams available to the GlooInstance. | | `upstreamGroups` | [.fed.solo.io.GlooInstanceSpec.Check.Summary](../instance.proto.sk/#summary) | UpstreamGroups describes the UpstreamGroups available to the GlooInstance. | | `proxies` | [.fed.solo.io.GlooInstanceSpec.Check.Summary](../instance.proto.sk/#summary) | Proxies describes the Proxies configuring the GlooInstance. | +| `rateLimitConfigs` | [.fed.solo.io.GlooInstanceSpec.Check.Summary](../instance.proto.sk/#summary) | RateLimitConfigs describes the RateLimitConfigs available to the GlooInstance. | +| `matchableHttpGateways` | [.fed.solo.io.GlooInstanceSpec.Check.Summary](../instance.proto.sk/#summary) | MatchableHttpGateways describes the MatchableHttpGateways available to the GlooInstance. | +| `matchableTcpGateways` | [.fed.solo.io.GlooInstanceSpec.Check.Summary](../instance.proto.sk/#summary) | MatchableTcpGateways describes the MatchableTcpGateways available to the GlooInstance. | | `deployments` | [.fed.solo.io.GlooInstanceSpec.Check.Summary](../instance.proto.sk/#summary) | Deployments describes the Deployments in the GlooInstance's install namespace. | | `pods` | [.fed.solo.io.GlooInstanceSpec.Check.Summary](../instance.proto.sk/#summary) | Pods describes the pods in the GlooInstance's install namespace. | diff --git a/go.mod b/go.mod index 56fbc9b2714..b8c1d41f71f 100644 --- a/go.mod +++ b/go.mod @@ -54,8 +54,8 @@ require ( github.com/solo-io/protoc-gen-openapi v0.2.2 github.com/solo-io/skv2 v0.36.5 - // Pinned to the latest `gloo-repo-branch` tag of solo-apis (`sa-k8s-1.28-bump`) - github.com/solo-io/solo-apis v0.0.0-20231206142556-d2e3ed6d4476 + // Pinned to the latest `gloo-repo-branch` tag of solo-apis (`gloo-v1.17.0-beta25`) + github.com/solo-io/solo-apis v0.0.0-20240503163415-57628abb2459 github.com/solo-io/solo-kit v0.34.2 github.com/spf13/afero v1.9.2 github.com/spf13/cobra v1.8.0 diff --git a/go.sum b/go.sum index 2659f49187d..6f24b324ea2 100644 --- a/go.sum +++ b/go.sum @@ -1983,8 +1983,8 @@ github.com/solo-io/protoc-gen-openapi v0.2.2 h1:OzyOAxiZuMAaLYWMNoTl2v9E7IXcl7UE github.com/solo-io/protoc-gen-openapi v0.2.2/go.mod h1:osEjRl1miHqlq4Wl/8SEqHFoyydptPL1EzEdM9c4vfE= github.com/solo-io/skv2 v0.36.5 h1:DV2mbY8vSQFXpmKN0KsKAagWHQWUP1sRjM42YGCAos4= github.com/solo-io/skv2 v0.36.5/go.mod h1:0GKILLOrQiTKvGsf7UEFZJiDxJHrp0yKJQ6NbHzn+vY= -github.com/solo-io/solo-apis v0.0.0-20231206142556-d2e3ed6d4476 h1:qtImo1deMtbVHKKkepwKfu1CDJBjGPvTmADtXVWBynE= -github.com/solo-io/solo-apis v0.0.0-20231206142556-d2e3ed6d4476/go.mod h1:/2NUdNZ37KAhD1AfwFGR54rOK7ldDvmro17c0YRriKk= +github.com/solo-io/solo-apis v0.0.0-20240503163415-57628abb2459 h1:fL/ZdBcFcxnQq5LZPP8h6eAMyrBai26JJqiCZ3j6HGo= +github.com/solo-io/solo-apis v0.0.0-20240503163415-57628abb2459/go.mod h1:QXQQr/+kzCA+/GR/jzPuUqQY+aaTU+9Xht3xWm6eAfk= github.com/solo-io/solo-kit v0.34.2 h1:qcQGvhGEBDY5zvDMcP7z4BjAi+v66BtNLm984hZQK/s= github.com/solo-io/solo-kit v0.34.2/go.mod h1:HkrxDRo+uR0FZUJi+SGAUkbrUNWC6V0FD7GGXbtC0UU= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= diff --git a/test/gomega/matchers/pod.go b/test/gomega/matchers/pod.go index 737df122597..730724c641c 100644 --- a/test/gomega/matchers/pod.go +++ b/test/gomega/matchers/pod.go @@ -9,9 +9,12 @@ import ( // ExpectedPod is a struct that represents the expected pod. type ExpectedPod struct { - // ContainerName is the name of the container. Required. + // ContainerName is the name of the container. Optional. ContainerName string + // Status is the pod phase status (e.g. Running, Pending, Succeeded, Failed). Optional. + Status corev1.PodPhase + // TODO(npolshak): Add more fields to match on as needed } @@ -29,18 +32,51 @@ func (pm *podMatcher) Match(actual interface{}) (bool, error) { if !ok { return false, fmt.Errorf("expected a pod, got %T", actual) } - for _, container := range pod.Spec.Containers { - if container.Name == pm.expectedPod.ContainerName { - return true, nil + if pm.expectedPod.ContainerName != "" { + foundContainer := false + for _, container := range pod.Spec.Containers { + if container.Name == pm.expectedPod.ContainerName { + foundContainer = true + } + } + if !foundContainer { + return false, nil + } + } + + if pm.expectedPod.Status != "" { + if pod.Status.Phase != pm.expectedPod.Status { + return false, nil } } - return false, nil + + return true, nil } func (pm *podMatcher) FailureMessage(actual interface{}) string { - return fmt.Sprintf("Expected pod to have container '%s', but it was not found", pm.expectedPod.ContainerName) + var errorMsg string + if pm.expectedPod.ContainerName != "" { + errorMsg += fmt.Sprintf("Expected pod to have container '%s', but it was not found", pm.expectedPod.ContainerName) + } + if pm.expectedPod.Status != "" { + errorMsg += fmt.Sprintf("Expected pod to have status '%s', but it was not found", pm.expectedPod.Status) + } + return errorMsg } func (pm *podMatcher) NegatedFailureMessage(actual interface{}) string { - return fmt.Sprintf("Expected pod not to have container '%s', but it was found", pm.expectedPod.ContainerName) + pod := actual.(corev1.Pod) + + var errorMsg string + if pm.expectedPod.ContainerName != "" { + containers := "" + for _, container := range pod.Spec.Containers { + containers += container.Name + ", " + } + errorMsg += fmt.Sprintf("Expected pod to have container '%s', but it found %s", pm.expectedPod.ContainerName, containers) + } + if pm.expectedPod.Status != "" { + errorMsg += fmt.Sprintf("Expected pod to have status '%s', but it found %s", pm.expectedPod.Status, pod.Status.Phase) + } + return errorMsg } diff --git a/test/kubernetes/e2e/features/istio/generate/generate.go b/test/kubernetes/e2e/features/istio/generate/generate.go index c052f4313f8..99ef15facb5 100644 --- a/test/kubernetes/e2e/features/istio/generate/generate.go +++ b/test/kubernetes/e2e/features/istio/generate/generate.go @@ -23,12 +23,36 @@ func main() { log.Println("starting generate for istio examples") // use the Gloo Edge Gateway api resources with automtls enabled - edgeGatewayApiResources := istio.GetGlooGatewayEdgeResources(exampleNs) - automtlsGeneratedExample := filepath.Join(util.MustGetThisDir(), "generated_example", fmt.Sprintf("automtls-enabled-%s", istio.EdgeApisRoutingResourcesFileName)) + edgeGatewayApiResources := istio.GetGlooGatewayEdgeResources(exampleNs, istio.UpstreamConfigOpts{}) + automtlsGeneratedExample := filepath.Join(util.MustGetThisDir(), "generated_example", fmt.Sprintf("automtls-enabled-%s", istio.EdgeApisRoutingFileName)) err := resources.WriteResourcesToFile(edgeGatewayApiResources, automtlsGeneratedExample) if err != nil { panic(err) } + // automtls disabled + edgeGatewayApiResources = istio.GetGlooGatewayEdgeResources(exampleNs, istio.UpstreamConfigOpts{DisableIstioAutoMtls: true}) + disableAutomtlsGeneratedExample := filepath.Join(util.MustGetThisDir(), "generated_example", fmt.Sprintf("automtls-disabled-%s", istio.EdgeApisRoutingFileName)) + err = resources.WriteResourcesToFile(edgeGatewayApiResources, disableAutomtlsGeneratedExample) + if err != nil { + panic(err) + } + + // Upstream sslConfig is set + edgeGatewayApiResources = istio.GetGlooGatewayEdgeResources(exampleNs, istio.UpstreamConfigOpts{SetSslConfig: true}) + upstreamSslConfigGeneratedExample := filepath.Join(util.MustGetThisDir(), "generated_example", fmt.Sprintf("sslconfig-%s", istio.EdgeApisRoutingFileName)) + err = resources.WriteResourcesToFile(edgeGatewayApiResources, upstreamSslConfigGeneratedExample) + if err != nil { + panic(err) + } + + // Upstream sslConfig is set and automtls is disabled + edgeGatewayApiResources = istio.GetGlooGatewayEdgeResources(exampleNs, istio.UpstreamConfigOpts{SetSslConfig: true, DisableIstioAutoMtls: true}) + sslConfigAndDisableAutomtlsGeneratedExample := filepath.Join(util.MustGetThisDir(), "generated_example", fmt.Sprintf("sslconfig-and-automtls-disabled-%s", istio.EdgeApisRoutingFileName)) + err = resources.WriteResourcesToFile(edgeGatewayApiResources, sslConfigAndDisableAutomtlsGeneratedExample) + if err != nil { + panic(err) + } + log.Println("finished generate for istio examples") } diff --git a/test/kubernetes/e2e/features/istio/generate/generated_example/automtls-disabled-edge-apis-routing.gen.yaml b/test/kubernetes/e2e/features/istio/generate/generated_example/automtls-disabled-edge-apis-routing.gen.yaml new file mode 100644 index 00000000000..ae07909b107 --- /dev/null +++ b/test/kubernetes/e2e/features/istio/generate/generated_example/automtls-disabled-edge-apis-routing.gen.yaml @@ -0,0 +1,34 @@ +apiVersion: gateway.solo.io/v1 +kind: VirtualService +metadata: + name: httpbin-vs + namespace: gloo-system +spec: + virtualHost: + domains: + - httpbin + routes: + - matchers: + - prefix: / + routeAction: + single: + upstream: + name: httpbin-upstream + namespace: gloo-system +status: {} +--- +apiVersion: gloo.solo.io/v1 +kind: Upstream +metadata: + name: httpbin-upstream + namespace: gloo-system +spec: + disableIstioAutoMtls: true + kube: + selector: + app: httpbin + serviceName: httpbin + serviceNamespace: httpbin + servicePort: 8000 +status: {} +--- diff --git a/test/kubernetes/e2e/features/istio/generate/generated_example/automtls-enabled-edge-apis-routing.gen.yaml b/test/kubernetes/e2e/features/istio/generate/generated_example/automtls-enabled-edge-apis-routing.gen.yaml index 6bd301e8e01..2de9f11b240 100644 --- a/test/kubernetes/e2e/features/istio/generate/generated_example/automtls-enabled-edge-apis-routing.gen.yaml +++ b/test/kubernetes/e2e/features/istio/generate/generated_example/automtls-enabled-edge-apis-routing.gen.yaml @@ -23,6 +23,7 @@ metadata: name: httpbin-upstream namespace: gloo-system spec: + disableIstioAutoMtls: false kube: selector: app: httpbin diff --git a/test/kubernetes/e2e/features/istio/generate/generated_example/sslconfig-and-automtls-disabled-edge-apis-routing.gen.yaml b/test/kubernetes/e2e/features/istio/generate/generated_example/sslconfig-and-automtls-disabled-edge-apis-routing.gen.yaml new file mode 100644 index 00000000000..a94c93d1072 --- /dev/null +++ b/test/kubernetes/e2e/features/istio/generate/generated_example/sslconfig-and-automtls-disabled-edge-apis-routing.gen.yaml @@ -0,0 +1,42 @@ +apiVersion: gateway.solo.io/v1 +kind: VirtualService +metadata: + name: httpbin-vs + namespace: gloo-system +spec: + virtualHost: + domains: + - httpbin + routes: + - matchers: + - prefix: / + routeAction: + single: + upstream: + name: httpbin-upstream + namespace: gloo-system +status: {} +--- +apiVersion: gloo.solo.io/v1 +kind: Upstream +metadata: + name: httpbin-upstream + namespace: gloo-system +spec: + disableIstioAutoMtls: true + kube: + selector: + app: httpbin + serviceName: httpbin + serviceNamespace: httpbin + servicePort: 8000 + sslConfig: + alpnProtocols: + - istio + sds: + certificatesSecretName: istio_server_cert + clusterName: gateway_proxy_sds + targetUri: 127.0.0.1:8234 + validationContextName: istio_validation_context +status: {} +--- diff --git a/test/kubernetes/e2e/features/istio/generate/generated_example/sslconfig-edge-apis-routing.gen.yaml b/test/kubernetes/e2e/features/istio/generate/generated_example/sslconfig-edge-apis-routing.gen.yaml new file mode 100644 index 00000000000..719a469b9e9 --- /dev/null +++ b/test/kubernetes/e2e/features/istio/generate/generated_example/sslconfig-edge-apis-routing.gen.yaml @@ -0,0 +1,42 @@ +apiVersion: gateway.solo.io/v1 +kind: VirtualService +metadata: + name: httpbin-vs + namespace: gloo-system +spec: + virtualHost: + domains: + - httpbin + routes: + - matchers: + - prefix: / + routeAction: + single: + upstream: + name: httpbin-upstream + namespace: gloo-system +status: {} +--- +apiVersion: gloo.solo.io/v1 +kind: Upstream +metadata: + name: httpbin-upstream + namespace: gloo-system +spec: + disableIstioAutoMtls: false + kube: + selector: + app: httpbin + serviceName: httpbin + serviceNamespace: httpbin + servicePort: 8000 + sslConfig: + alpnProtocols: + - istio + sds: + certificatesSecretName: istio_server_cert + clusterName: gateway_proxy_sds + targetUri: 127.0.0.1:8234 + validationContextName: istio_validation_context +status: {} +--- diff --git a/test/kubernetes/e2e/features/istio/gloo_gateway_auto_mtls_suite.go b/test/kubernetes/e2e/features/istio/gloo_gateway_auto_mtls_suite.go index 0d00c3ac9e0..9183e8413cd 100644 --- a/test/kubernetes/e2e/features/istio/gloo_gateway_auto_mtls_suite.go +++ b/test/kubernetes/e2e/features/istio/gloo_gateway_auto_mtls_suite.go @@ -2,8 +2,11 @@ package istio import ( "context" + "fmt" "path/filepath" + "time" + "github.com/onsi/gomega" "github.com/solo-io/gloo/pkg/utils/kubeutils" "github.com/solo-io/gloo/pkg/utils/requestutils/curl" "github.com/solo-io/gloo/projects/gateway/pkg/defaults" @@ -23,41 +26,57 @@ type glooIstioAutoMtlsTestingSuite struct { // against an installation of Gloo Gateway testInstallation *e2e.TestInstallation - // routingManifestPath is the path to the manifest directory that contains the routing resources - routingManifestPath string + // generated routing manifest file names + enableAutomtlsFile string + disableAutomtlsFile string + sslConfigFile string + sslConfigAndDisableAutomtlsFile string } func NewGlooIstioAutoMtlsSuite(ctx context.Context, testInst *e2e.TestInstallation) suite.TestingSuite { + enableAutomtlsFile := filepath.Join(testInst.GeneratedFiles.TempDir, fmt.Sprintf("glooIstioAutoMtlsTestingSuite-%s", getGlooGatewayEdgeResourceFilmeName(UpstreamConfigOpts{}))) + disableAutomtlsFile := filepath.Join(testInst.GeneratedFiles.TempDir, fmt.Sprintf("glooIstioAutoMtlsTestingSuite-%s", getGlooGatewayEdgeResourceFilmeName(UpstreamConfigOpts{DisableIstioAutoMtls: true}))) + sslConfigFile := filepath.Join(testInst.GeneratedFiles.TempDir, fmt.Sprintf("glooIstioAutoMtlsTestingSuite-%s", getGlooGatewayEdgeResourceFilmeName(UpstreamConfigOpts{SetSslConfig: true}))) + sslConfigAndDisableAutomtlsFile := filepath.Join(testInst.GeneratedFiles.TempDir, fmt.Sprintf("glooIstioAutoMtlsTestingSuite-%s", getGlooGatewayEdgeResourceFilmeName(UpstreamConfigOpts{SetSslConfig: true, DisableIstioAutoMtls: true}))) + return &glooIstioAutoMtlsTestingSuite{ - ctx: ctx, - testInstallation: testInst, - routingManifestPath: testInst.GeneratedFiles.TempDir, + ctx: ctx, + testInstallation: testInst, + enableAutomtlsFile: enableAutomtlsFile, + disableAutomtlsFile: disableAutomtlsFile, + sslConfigFile: sslConfigFile, + sslConfigAndDisableAutomtlsFile: sslConfigAndDisableAutomtlsFile, } } -func (s *glooIstioAutoMtlsTestingSuite) getEdgeGatewayRoutingManifest() string { - // TODO(npolshak): Support other upstream configurations (auto mtls disabled, sslConfig overwrite, etc.) - return filepath.Join(s.routingManifestPath, EdgeApisRoutingResourcesFileName) -} - func (s *glooIstioAutoMtlsTestingSuite) SetupSuite() { - gwResources := GetGlooGatewayEdgeResources(s.testInstallation.Metadata.InstallNamespace) - err := resources.WriteResourcesToFile(gwResources, s.getEdgeGatewayRoutingManifest()) - s.NoError(err, "can write resources to file") - - err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, setupManifest) + err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, setupManifest) s.NoError(err, "can apply setup manifest") + s.testInstallation.Assertions.EventuallyRunningReplicas(s.ctx, httpbinDeployment.ObjectMeta, gomega.Equal(1)) - // Ensure that the proxy service and deployment are created - err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, s.getEdgeGatewayRoutingManifest()) - s.NoError(err, "can apply generated routing manifest") + // enabled automtls on upstream + enableAutomtlsResources := GetGlooGatewayEdgeResources(s.testInstallation.Metadata.InstallNamespace, UpstreamConfigOpts{}) + err = resources.WriteResourcesToFile(enableAutomtlsResources, s.enableAutomtlsFile) + s.NoError(err, "can write automtls upstream resources to file") + + // disable automtls on upstream + disableAutomtlsResources := GetGlooGatewayEdgeResources(s.testInstallation.Metadata.InstallNamespace, UpstreamConfigOpts{DisableIstioAutoMtls: true}) + err = resources.WriteResourcesToFile(disableAutomtlsResources, s.disableAutomtlsFile) + s.NoError(err, "can write disabled automtls upstream resources to file") + + // sslConfig and automtls on upstream + sslConfigResources := GetGlooGatewayEdgeResources(s.testInstallation.Metadata.InstallNamespace, UpstreamConfigOpts{SetSslConfig: true}) + err = resources.WriteResourcesToFile(sslConfigResources, s.sslConfigFile) + s.NoError(err, "can write sslConfig automtls upstream resources to file") + + // sslConfig and disable automtls on upstream + sslConfigAndDisableAutomtlsResources := GetGlooGatewayEdgeResources(s.testInstallation.Metadata.InstallNamespace, UpstreamConfigOpts{SetSslConfig: true, DisableIstioAutoMtls: true}) + err = resources.WriteResourcesToFile(sslConfigAndDisableAutomtlsResources, s.sslConfigAndDisableAutomtlsFile) + s.NoError(err, "can write sslConfig and disable automtls upstream resources to file") } func (s *glooIstioAutoMtlsTestingSuite) TearDownSuite() { - err := s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, s.getEdgeGatewayRoutingManifest()) - s.NoError(err, "can delete generated routing manifest") - - err = s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, setupManifest) + err := s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, setupManifest) s.NoError(err, "can delete setup manifest") } @@ -65,9 +84,17 @@ func (s *glooIstioAutoMtlsTestingSuite) TestMtlsStrictPeerAuth() { s.T().Cleanup(func() { err := s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, strictPeerAuthManifest) s.NoError(err, "can delete manifest") + + err = s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, s.enableAutomtlsFile) + s.NoError(err, "can delete generated routing manifest") }) - err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, strictPeerAuthManifest) + // Ensure that the proxy service and deployment are created + err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, s.enableAutomtlsFile) + s.NoError(err, "can apply generated routing manifest") + + // Apply strict peer auth policy + err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, strictPeerAuthManifest) s.NoError(err, "can apply strictPeerAuthManifest") s.testInstallation.Assertions.AssertEventualCurlResponse( @@ -79,16 +106,36 @@ func (s *glooIstioAutoMtlsTestingSuite) TestMtlsStrictPeerAuth() { curl.WithPath("headers"), curl.WithPort(80), }, - expectedMtlsResponse) + expectedMtlsResponse, time.Minute) } func (s *glooIstioAutoMtlsTestingSuite) TestMtlsPermissivePeerAuth() { s.T().Cleanup(func() { err := s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, permissivePeerAuthManifest) s.NoError(err, "can delete manifest") + + err = s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, s.enableAutomtlsFile) + s.NoError(err, "can delete generated routing manifest") }) - err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, permissivePeerAuthManifest) + // Initially use automtls (no sslConfig on upstream) + err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, s.enableAutomtlsFile) + // Ensure that the proxy service and deployment are created + s.NoError(err, "can apply generated routing manifest") + + s.testInstallation.Assertions.AssertEventualCurlResponse( + s.ctx, + curlPodExecOpt, + []curl.Option{ + curl.WithHost(kubeutils.ServiceFQDN(metav1.ObjectMeta{Name: defaults.GatewayProxyName, Namespace: s.testInstallation.Metadata.InstallNamespace})), + curl.WithHostHeader("httpbin"), + curl.WithPath("/headers"), + curl.WithPort(80), + }, + expectedMtlsResponse, time.Minute) + + // Apply permissive peer auth policy + err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, permissivePeerAuthManifest) s.NoError(err, "can apply permissivePeerAuth") // With auto mtls enabled in the mesh, the response should contain the X-Forwarded-Client-Cert header even with permissive mode @@ -101,5 +148,175 @@ func (s *glooIstioAutoMtlsTestingSuite) TestMtlsPermissivePeerAuth() { curl.WithPath("headers"), curl.WithPort(80), }, - expectedMtlsResponse) + expectedMtlsResponse, time.Minute) +} + +func (s *glooIstioAutoMtlsTestingSuite) TestMtlsDisablePeerAuth() { + s.T().Cleanup(func() { + err := s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, disablePeerAuthManifest) + s.NoError(err, "can delete manifest") + + // Routing with k8s svc as the destination + err = s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, s.disableAutomtlsFile) + s.NoError(err, "can delete generated routing manifest") + }) + + // Apply routing config + err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, s.disableAutomtlsFile) + s.NoError(err, "can apply generated routing manifest") + + // Apply disable peer auth Istio policy + err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, disablePeerAuthManifest) + s.NoError(err, "can apply disablePeerAuthManifest") + + s.testInstallation.Assertions.AssertEventualCurlResponse( + s.ctx, + curlPodExecOpt, + []curl.Option{ + curl.WithHost(kubeutils.ServiceFQDN(metav1.ObjectMeta{Name: defaults.GatewayProxyName, Namespace: s.testInstallation.Metadata.InstallNamespace})), + curl.WithHostHeader("httpbin"), + curl.WithPath("/headers"), + curl.WithPort(80), + }, + expectedPlaintextResponse, time.Minute) +} + +func (s *glooIstioAutoMtlsTestingSuite) TestUpgrade() { + s.T().Cleanup(func() { + // Clean up peer auth + err := s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, strictPeerAuthManifest) + s.NoError(err, "can delete manifest") + + // Clean up the final resources + err = s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, s.enableAutomtlsFile) + s.NoError(err, "can delete manifest") + }) + + // Initially use sslConfig on upstream + err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, s.disableAutomtlsFile) + s.NoError(err, "can apply generated routing manifest with sslConfig upstream") + + // Apply strict peer auth Istio policy to check traffic is consistently mtls + err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, strictPeerAuthManifest) + s.NoError(err, "can apply strictPeerAuthManifest") + + // Check sslConfig upstream is working + s.testInstallation.Assertions.AssertEventualCurlResponse( + s.ctx, + curlPodExecOpt, + []curl.Option{ + curl.WithHost(kubeutils.ServiceFQDN(metav1.ObjectMeta{Name: defaults.GatewayProxyName, Namespace: s.testInstallation.Metadata.InstallNamespace})), + curl.WithHostHeader("httpbin"), + curl.WithPath("/headers"), + curl.WithPort(80), + }, + expectedMtlsResponse, time.Minute) + + // Switch to automtls (remove sslConfig on upstream) + err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, s.enableAutomtlsFile) + s.NoError(err, "can apply generated routing manifest with automtls upstream") + + // Check sslConfig upstream is working + s.testInstallation.Assertions.AssertEventualCurlResponse( + s.ctx, + curlPodExecOpt, + []curl.Option{ + curl.WithHost(kubeutils.ServiceFQDN(metav1.ObjectMeta{Name: defaults.GatewayProxyName, Namespace: s.testInstallation.Metadata.InstallNamespace})), + curl.WithHostHeader("httpbin"), + curl.WithPath("/headers"), + curl.WithPort(80), + }, + expectedMtlsResponse, time.Minute) +} + +func (s *glooIstioAutoMtlsTestingSuite) TestDowngrade() { + s.T().Cleanup(func() { + // Clean up peer auth + err := s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, strictPeerAuthManifest) + s.NoError(err, "can delete manifest") + + // Clean up the final resources + err = s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, s.sslConfigFile) + s.NoError(err, "can delete manifest") + }) + + // Initially use automtls (remove sslConfig on upstream) + err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, s.enableAutomtlsFile) + s.NoError(err, "can apply generated routing manifest with automtls upstream") + + // Apply strict peer auth Istio policy to check traffic is consistently mtls + err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, strictPeerAuthManifest) + s.NoError(err, "can apply strictPeerAuthManifest") + + // Check sslConfig upstream is working + s.testInstallation.Assertions.AssertEventualCurlResponse( + s.ctx, + curlPodExecOpt, + []curl.Option{ + curl.WithHost(kubeutils.ServiceFQDN(metav1.ObjectMeta{Name: defaults.GatewayProxyName, Namespace: s.testInstallation.Metadata.InstallNamespace})), + curl.WithHostHeader("httpbin"), + curl.WithPath("/headers"), + curl.WithPort(80), + }, + expectedMtlsResponse, time.Minute) + + // Switch to use sslConfig on upstream (do not explicitly disable automtls) + err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, s.sslConfigFile) + s.NoError(err, "can apply generated routing manifest with sslConfig upstream") + + // Check sslConfig upstream is working + s.testInstallation.Assertions.AssertEventualCurlResponse( + s.ctx, + curlPodExecOpt, + []curl.Option{ + curl.WithHost(kubeutils.ServiceFQDN(metav1.ObjectMeta{Name: defaults.GatewayProxyName, Namespace: s.testInstallation.Metadata.InstallNamespace})), + curl.WithHostHeader("httpbin"), + curl.WithPath("/headers"), + curl.WithPort(80), + }, + expectedMtlsResponse, time.Minute) +} + +func (s *glooIstioAutoMtlsTestingSuite) TestDisableAutomtlsOverridesSSLConfig() { + s.T().Cleanup(func() { + // Clean up peer auth + err := s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, disablePeerAuthManifest) + s.NoError(err, "can delete manifest") + + // Clean up the final resources + err = s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, s.sslConfigAndDisableAutomtlsFile) + s.NoError(err, "can delete manifest") + }) + + // Uuse sslConfig on upstream with automtls disabled (sslConfig will overwrite automtls) + err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, s.sslConfigAndDisableAutomtlsFile) + s.NoError(err, "can apply generated routing manifest with sslConfig upstream") + + // Check sslConfig upstream is working + s.testInstallation.Assertions.AssertEventualCurlResponse( + s.ctx, + curlPodExecOpt, + []curl.Option{ + curl.WithHost(kubeutils.ServiceFQDN(metav1.ObjectMeta{Name: defaults.GatewayProxyName, Namespace: s.testInstallation.Metadata.InstallNamespace})), + curl.WithHostHeader("httpbin"), + curl.WithPath("/headers"), + curl.WithPort(80), + }, + expectedMtlsResponse, time.Minute) + + // Apply disable peer auth + err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, disablePeerAuthManifest) + s.NoError(err, "can apply disablePeerAuthManifest") + + // Check disable peer auth policy is working when sslConfig is set + s.testInstallation.Assertions.AssertEventualCurlResponse( + s.ctx, + curlPodExecOpt, + []curl.Option{ + curl.WithHost(kubeutils.ServiceFQDN(metav1.ObjectMeta{Name: defaults.GatewayProxyName, Namespace: s.testInstallation.Metadata.InstallNamespace})), + curl.WithHostHeader("httpbin"), + curl.WithPath("/headers"), + curl.WithPort(80), + }, + expectedServiceUnavailableResponse, time.Minute) } diff --git a/test/kubernetes/e2e/features/istio/gloo_gateway_no_auto_mtls_suite.go b/test/kubernetes/e2e/features/istio/gloo_gateway_no_auto_mtls_suite.go index 65ca42a2a7c..3ab79659574 100644 --- a/test/kubernetes/e2e/features/istio/gloo_gateway_no_auto_mtls_suite.go +++ b/test/kubernetes/e2e/features/istio/gloo_gateway_no_auto_mtls_suite.go @@ -2,13 +2,16 @@ package istio import ( "context" + "fmt" "path/filepath" + "time" "github.com/solo-io/gloo/pkg/utils/kubeutils" "github.com/solo-io/gloo/pkg/utils/requestutils/curl" "github.com/solo-io/gloo/projects/gateway/pkg/defaults" "github.com/solo-io/gloo/test/kubernetes/e2e" "github.com/solo-io/gloo/test/kubernetes/testutils/resources" + "github.com/solo-io/go-utils/contextutils" "github.com/stretchr/testify/suite" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -24,48 +27,80 @@ type glooIstioTestingSuite struct { // against an installation of Gloo Gateway testInstallation *e2e.TestInstallation - // routingManifestFile is the path to the manifest file that contains the routing resources - routingManifestFile string + // maps test name to a list of manifests to apply before the test + manifests map[string][]string } func NewGlooTestingSuite(ctx context.Context, testInst *e2e.TestInstallation) suite.TestingSuite { - routingManifestFile := filepath.Join(testInst.GeneratedFiles.TempDir, EdgeApisRoutingResourcesFileName) + log := contextutils.LoggerFrom(ctx) + + noMtlsGlooResourcesFile := filepath.Join(testInst.GeneratedFiles.TempDir, fmt.Sprintf("glooIstioTestingSuite-%s", getGlooGatewayEdgeResourceFilmeName(UpstreamConfigOpts{}))) + sslGlooResourcesFile := filepath.Join(testInst.GeneratedFiles.TempDir, fmt.Sprintf("glooIstioTestingSuite-%s", getGlooGatewayEdgeResourceFilmeName(UpstreamConfigOpts{SetSslConfig: true}))) + + noMtlsResources := GetGlooGatewayEdgeResources(testInst.Metadata.InstallNamespace, UpstreamConfigOpts{}) + err := resources.WriteResourcesToFile(noMtlsResources, noMtlsGlooResourcesFile) + if err != nil { + log.Error(err, "can write resources to file") + } + + sslResources := GetGlooGatewayEdgeResources(testInst.Metadata.InstallNamespace, UpstreamConfigOpts{SetSslConfig: true}) + err = resources.WriteResourcesToFile(sslResources, sslGlooResourcesFile) + if err != nil { + log.Error(err, "can write resources to file") + } + return &glooIstioTestingSuite{ - ctx: ctx, - testInstallation: testInst, - routingManifestFile: routingManifestFile, + ctx: ctx, + testInstallation: testInst, + manifests: map[string][]string{ + "TestStrictPeerAuth": {strictPeerAuthManifest, noMtlsGlooResourcesFile}, + "TestPermissivePeerAuth": {permissivePeerAuthManifest, noMtlsGlooResourcesFile}, + "TestUpstreamSSLConfigStrictPeerAuth": {strictPeerAuthManifest, sslGlooResourcesFile}, + }, } } func (s *glooIstioTestingSuite) SetupSuite() { - gwResources := GetGlooGatewayEdgeResources(s.testInstallation.Metadata.InstallNamespace) - err := resources.WriteResourcesToFile(gwResources, s.routingManifestFile) - s.NoError(err, "can write resources to file") - - err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, setupManifest) + err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, setupManifest) s.NoError(err, "can apply setup manifest") - - err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, s.routingManifestFile) - s.NoError(err, "can apply generated manifest") + // Check that istio injection is successful and httpbin is running + s.testInstallation.Assertions.EventuallyObjectsExist(s.ctx, httpbinDeployment) + // httpbin can take a while to start up with Istio sidecar + s.testInstallation.Assertions.EventuallyPodsRunning(s.ctx, httpbinDeployment.ObjectMeta.GetNamespace(), + metav1.ListOptions{LabelSelector: "app=httpbin"}, time.Minute*2) } func (s *glooIstioTestingSuite) TearDownSuite() { - err := s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, s.routingManifestFile) - s.NoError(err, "can delete generated routing manifest") - - err = s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, setupManifest) + err := s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, setupManifest) s.NoError(err, "can delete setup manifest") + s.testInstallation.Assertions.EventuallyObjectsNotExist(s.ctx, httpbinDeployment) } -func (s *glooIstioTestingSuite) TestStrictPeerAuth() { - s.T().Cleanup(func() { - err := s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, strictPeerAuthManifest) - s.NoError(err, "can delete manifest") - }) +func (s *glooIstioTestingSuite) BeforeTest(suiteName, testName string) { + manifests, ok := s.manifests[testName] + if !ok { + s.FailNow("no manifests found for %s, manifest map contents: %v", testName, s.manifests) + } + + for _, manifest := range manifests { + err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, manifest) + s.NoError(err, "can apply "+manifest) + } +} - err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, strictPeerAuthManifest) - s.NoError(err, "can apply strictPeerAuthManifest") +func (s *glooIstioTestingSuite) AfterTest(suiteName, testName string) { + manifests, ok := s.manifests[testName] + if !ok { + s.FailNow("no manifests found for " + testName) + } + for _, manifest := range manifests { + err := s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, manifest) + s.NoError(err, "can delete "+manifest) + } +} + +func (s *glooIstioTestingSuite) TestStrictPeerAuth() { // With auto mtls disabled in the mesh, the request should fail when the strict peer auth policy is applied s.testInstallation.Assertions.AssertEventualCurlResponse( s.ctx, @@ -76,18 +111,10 @@ func (s *glooIstioTestingSuite) TestStrictPeerAuth() { curl.WithPath("headers"), curl.WithPort(80), }, - expectedServiceUnavailableResponse) + expectedServiceUnavailableResponse, time.Minute) } func (s *glooIstioTestingSuite) TestPermissivePeerAuth() { - s.T().Cleanup(func() { - err := s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, permissivePeerAuthManifest) - s.NoError(err, "can delete manifest") - }) - - err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, permissivePeerAuthManifest) - s.NoError(err, "can apply permissivePeerAuth") - // With auto mtls disabled in the mesh, the response should not contain the X-Forwarded-Client-Cert header s.testInstallation.Assertions.AssertEventualCurlResponse( s.ctx, @@ -98,5 +125,19 @@ func (s *glooIstioTestingSuite) TestPermissivePeerAuth() { curl.WithPath("headers"), curl.WithPort(80), }, - expectedPlaintextResponse) + expectedPlaintextResponse, time.Minute) +} + +func (s *glooIstioTestingSuite) TestUpstreamSSLConfigStrictPeerAuth() { + // With auto mtls disabled in the mesh, the request should succeed when Upstream is configured with sslConfig + s.testInstallation.Assertions.AssertEventualCurlResponse( + s.ctx, + curlPodExecOpt, + []curl.Option{ + curl.WithHost(kubeutils.ServiceFQDN(metav1.ObjectMeta{Name: defaults.GatewayProxyName, Namespace: s.testInstallation.Metadata.InstallNamespace})), + curl.WithHostHeader("httpbin"), + curl.WithPath("headers"), + curl.WithPort(80), + }, + expectedMtlsResponse, time.Minute) } diff --git a/test/kubernetes/e2e/features/istio/k8s_gateway_auto_mtls_suite.go b/test/kubernetes/e2e/features/istio/k8s_gateway_auto_mtls_suite.go index 46d6dd028a8..489685cdcf4 100644 --- a/test/kubernetes/e2e/features/istio/k8s_gateway_auto_mtls_suite.go +++ b/test/kubernetes/e2e/features/istio/k8s_gateway_auto_mtls_suite.go @@ -2,9 +2,10 @@ package istio import ( "context" + "time" - "github.com/onsi/gomega" "github.com/stretchr/testify/suite" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/solo-io/gloo/pkg/utils/kubeutils" "github.com/solo-io/gloo/pkg/utils/requestutils/curl" @@ -20,6 +21,9 @@ type istioAutoMtlsTestingSuite struct { // testInstallation contains all the metadata/utilities necessary to execute a series of tests // against an installation of Gloo Gateway testInstallation *e2e.TestInstallation + + // maps test name to a list of manifests to apply before the test + manifests map[string][]string } func NewIstioAutoMtlsSuite(ctx context.Context, testInst *e2e.TestInstallation) suite.TestingSuite { @@ -29,37 +33,60 @@ func NewIstioAutoMtlsSuite(ctx context.Context, testInst *e2e.TestInstallation) } } +func (s *istioAutoMtlsTestingSuite) BeforeTest(suiteName, testName string) { + manifests, ok := s.manifests[testName] + if !ok { + s.FailNow("no manifests found for %s, manifest map contents: %v", testName, s.manifests) + } + + for _, manifest := range manifests { + err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, manifest) + s.NoError(err, "can apply "+manifest) + } + + s.testInstallation.Assertions.EventuallyObjectsExist(s.ctx, proxyService, proxyDeployment) + // Check that test resources are running + s.testInstallation.Assertions.EventuallyPodsRunning(s.ctx, proxyDeployment.ObjectMeta.GetNamespace(), + metav1.ListOptions{LabelSelector: "app.kubernetes.io/name=gloo-proxy-gw"}, time.Minute*2) +} + +func (s *istioAutoMtlsTestingSuite) AfterTest(suiteName, testName string) { + manifests, ok := s.manifests[testName] + if !ok { + s.FailNow("no manifests found for " + testName) + } + + for _, manifest := range manifests { + err := s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, manifest) + s.NoError(err, "can delete "+manifest) + } + + s.testInstallation.Assertions.EventuallyObjectsNotExist(s.ctx, proxyService, proxyDeployment) +} + func (s *istioAutoMtlsTestingSuite) SetupSuite() { err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, setupManifest) s.NoError(err, "can apply setup manifest") // Check that istio injection is successful and httpbin is running - s.testInstallation.Assertions.EventuallyRunningReplicas(s.ctx, httpbinDeployment.ObjectMeta, gomega.Equal(1)) - - // Ensure that the proxy service and deployment are created - err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, k8sRoutingManifest) - s.NoError(err, "can apply k8s routing manifest") - s.testInstallation.Assertions.EventuallyObjectsExist(s.ctx, proxyService, proxyDeployment) + s.testInstallation.Assertions.EventuallyObjectsExist(s.ctx, httpbinDeployment) + s.testInstallation.Assertions.EventuallyPodsRunning(s.ctx, httpbinDeployment.ObjectMeta.GetNamespace(), + metav1.ListOptions{LabelSelector: "app=httpbin"}, time.Minute*2) + + // We include tests with manual setup here because the cleanup is still automated via AfterTest + s.manifests = map[string][]string{ + "TestMtlsStrictPeerAuth": {strictPeerAuthManifest, k8sRoutingSvcManifest}, + "TestMtlsPermissivePeerAuth": {permissivePeerAuthManifest, k8sRoutingSvcManifest}, + "TestMtlsDisablePeerAuth": {disablePeerAuthManifest, k8sRoutingUpstreamManifest}, + } } func (s *istioAutoMtlsTestingSuite) TearDownSuite() { - err := s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, k8sRoutingManifest) - s.NoError(err, "can delete k8s routing manifest") - s.testInstallation.Assertions.EventuallyObjectsNotExist(s.ctx, proxyService, proxyDeployment) - - err = s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, setupManifest) + err := s.testInstallation.Actions.Kubectl().DeleteFileSafe(s.ctx, setupManifest) s.NoError(err, "can delete setup manifest") s.testInstallation.Assertions.EventuallyObjectsNotExist(s.ctx, httpbinDeployment) } func (s *istioAutoMtlsTestingSuite) TestMtlsStrictPeerAuth() { - s.T().Cleanup(func() { - err := s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, strictPeerAuthManifest) - s.NoError(err, "can delete manifest") - }) - - err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, strictPeerAuthManifest) - s.NoError(err, "can apply strictPeerAuthManifest") - s.testInstallation.Assertions.AssertEventualCurlResponse( s.ctx, curlPodExecOpt, @@ -68,19 +95,10 @@ func (s *istioAutoMtlsTestingSuite) TestMtlsStrictPeerAuth() { curl.WithHostHeader("httpbin"), curl.WithPath("headers"), }, - expectedMtlsResponse, - ) + expectedMtlsResponse, time.Minute) } func (s *istioAutoMtlsTestingSuite) TestMtlsPermissivePeerAuth() { - s.T().Cleanup(func() { - err := s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, permissivePeerAuthManifest) - s.NoError(err, "can delete manifest") - }) - - err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, permissivePeerAuthManifest) - s.NoError(err, "can apply permissivePeerAuth") - // With auto mtls enabled in the mesh, the response should contain the X-Forwarded-Client-Cert header even with permissive mode s.testInstallation.Assertions.AssertEventualCurlResponse( s.ctx, @@ -90,5 +108,17 @@ func (s *istioAutoMtlsTestingSuite) TestMtlsPermissivePeerAuth() { curl.WithHostHeader("httpbin"), curl.WithPath("headers"), }, - expectedMtlsResponse) + expectedMtlsResponse, time.Minute) +} + +func (s *istioAutoMtlsTestingSuite) TestMtlsDisablePeerAuth() { + s.testInstallation.Assertions.AssertEventualCurlResponse( + s.ctx, + curlPodExecOpt, + []curl.Option{ + curl.WithHost(kubeutils.ServiceFQDN(proxyService.ObjectMeta)), + curl.WithHostHeader("httpbin"), + curl.WithPath("headers"), + }, + expectedPlaintextResponse, time.Minute) } diff --git a/test/kubernetes/e2e/features/istio/k8s_gateway_no_auto_mtls_suite.go b/test/kubernetes/e2e/features/istio/k8s_gateway_no_auto_mtls_suite.go index 2a5f20c32c0..0ec8dcb5fa2 100644 --- a/test/kubernetes/e2e/features/istio/k8s_gateway_no_auto_mtls_suite.go +++ b/test/kubernetes/e2e/features/istio/k8s_gateway_no_auto_mtls_suite.go @@ -2,12 +2,13 @@ package istio import ( "context" + "time" - "github.com/onsi/gomega" "github.com/solo-io/gloo/pkg/utils/kubeutils" "github.com/solo-io/gloo/pkg/utils/requestutils/curl" "github.com/solo-io/gloo/test/kubernetes/e2e" "github.com/stretchr/testify/suite" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // istioTestingSuite is the entire Suite of tests for the "Istio" integration cases where auto mtls is disabled @@ -20,6 +21,40 @@ type istioTestingSuite struct { // testInstallation contains all the metadata/utilities necessary to execute a series of tests // against an installation of Gloo Gateway testInstallation *e2e.TestInstallation + + // maps test name to a list of manifests to apply before the test + manifests map[string][]string +} + +func (s *istioTestingSuite) BeforeTest(suiteName, testName string) { + manifests, ok := s.manifests[testName] + if !ok { + s.FailNow("no manifests found for %s, manifest map contents: %v", testName, s.manifests) + } + + for _, manifest := range manifests { + err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, manifest) + s.NoError(err, "can apply "+manifest) + } + + s.testInstallation.Assertions.EventuallyObjectsExist(s.ctx, proxyService, proxyDeployment) + // Check that test resources are running. This can take a little longer for Istio tests due to the istio-proxy and sds sidecars + s.testInstallation.Assertions.EventuallyPodsRunning(s.ctx, proxyDeployment.ObjectMeta.GetNamespace(), + metav1.ListOptions{LabelSelector: "app.kubernetes.io/name=gloo-proxy-gw"}, time.Minute*2) +} + +func (s *istioTestingSuite) AfterTest(suiteName, testName string) { + manifests, ok := s.manifests[testName] + if !ok { + s.FailNow("no manifests found for " + testName) + } + + for _, manifest := range manifests { + err := s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, manifest) + s.NoError(err, "can delete "+manifest) + } + + s.testInstallation.Assertions.EventuallyObjectsNotExist(s.ctx, proxyService, proxyDeployment) } func NewTestingSuite(ctx context.Context, testInst *e2e.TestInstallation) suite.TestingSuite { @@ -33,34 +68,25 @@ func (s *istioTestingSuite) SetupSuite() { err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, setupManifest) s.NoError(err, "can apply setup manifest") // Check that istio injection is successful and httpbin is running - s.testInstallation.Assertions.EventuallyRunningReplicas(s.ctx, httpbinDeployment.ObjectMeta, gomega.Equal(1)) - - // Ensure that the proxy service and deployment are created - err = s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, k8sRoutingManifest) - s.NoError(err, "can apply k8s routing manifest") - s.testInstallation.Assertions.EventuallyObjectsExist(s.ctx, proxyService, proxyDeployment) + s.testInstallation.Assertions.EventuallyObjectsExist(s.ctx, httpbinDeployment) + // httpbin can take a while to start up with Istio sidecar + s.testInstallation.Assertions.EventuallyPodsRunning(s.ctx, httpbinDeployment.ObjectMeta.GetNamespace(), + metav1.ListOptions{LabelSelector: "app=httpbin"}, time.Minute*2) + + // We include tests with manual setup here because the cleanup is still automated via AfterTest + s.manifests = map[string][]string{ + "TestStrictPeerAuth": {strictPeerAuthManifest, k8sRoutingSvcManifest}, + "TestPermissivePeerAuth": {permissivePeerAuthManifest, k8sRoutingSvcManifest}, + } } func (s *istioTestingSuite) TearDownSuite() { - err := s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, k8sRoutingManifest) - s.NoError(err, "can delete k8s routing manifest") - s.testInstallation.Assertions.EventuallyObjectsNotExist(s.ctx, proxyService, proxyDeployment) - - err = s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, setupManifest) + err := s.testInstallation.Actions.Kubectl().DeleteFileSafe(s.ctx, setupManifest) s.NoError(err, "can delete setup manifest") s.testInstallation.Assertions.EventuallyObjectsNotExist(s.ctx, httpbinDeployment) - } func (s *istioTestingSuite) TestStrictPeerAuth() { - s.T().Cleanup(func() { - err := s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, strictPeerAuthManifest) - s.NoError(err, "can delete manifest") - }) - - err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, strictPeerAuthManifest) - s.NoError(err, "can apply strictPeerAuthManifest") - // With auto mtls disabled in the mesh, the request should fail when the strict peer auth policy is applied s.testInstallation.Assertions.AssertEventualCurlResponse( s.ctx, @@ -70,19 +96,10 @@ func (s *istioTestingSuite) TestStrictPeerAuth() { curl.WithHostHeader("httpbin"), curl.WithPath("headers"), }, - expectedServiceUnavailableResponse, - ) + expectedServiceUnavailableResponse, time.Minute) } func (s *istioTestingSuite) TestPermissivePeerAuth() { - s.T().Cleanup(func() { - err := s.testInstallation.Actions.Kubectl().DeleteFile(s.ctx, permissivePeerAuthManifest) - s.NoError(err, "can delete manifest") - }) - - err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, permissivePeerAuthManifest) - s.NoError(err, "can apply permissivePeerAuth") - // With auto mtls disabled in the mesh, the response should not contain the X-Forwarded-Client-Cert header s.testInstallation.Assertions.AssertEventualCurlResponse( s.ctx, @@ -92,5 +109,5 @@ func (s *istioTestingSuite) TestPermissivePeerAuth() { curl.WithHostHeader("httpbin"), curl.WithPath("headers"), }, - expectedPlaintextResponse) + expectedPlaintextResponse, time.Minute) } diff --git a/test/kubernetes/e2e/features/istio/resources.go b/test/kubernetes/e2e/features/istio/resources.go index 62b7f6a0759..453f9efb25e 100644 --- a/test/kubernetes/e2e/features/istio/resources.go +++ b/test/kubernetes/e2e/features/istio/resources.go @@ -3,24 +3,76 @@ package istio import ( "fmt" + "github.com/golang/protobuf/ptypes/wrappers" + "github.com/solo-io/gloo/projects/gloo/constants" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + gloov1 "github.com/solo-io/gloo/projects/gloo/pkg/api/v1" v1 "github.com/solo-io/solo-apis/pkg/api/gateway.solo.io/v1" soloapis_gloov1 "github.com/solo-io/solo-apis/pkg/api/gloo.solo.io/v1" "github.com/solo-io/solo-apis/pkg/api/gloo.solo.io/v1/core/matchers" soloapis_kubernetes "github.com/solo-io/solo-apis/pkg/api/gloo.solo.io/v1/options/kubernetes" + "github.com/solo-io/solo-apis/pkg/api/gloo.solo.io/v1/ssl" gloocore "github.com/solo-io/solo-kit/pkg/api/v1/resources/core" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" ) +type UpstreamConfigOpts struct { + DisableIstioAutoMtls bool + SetSslConfig bool +} + var ( - EdgeApisRoutingResourcesFileName = "edge-apis-routing.gen.yaml" + EdgeApisRoutingFileName = "edge-apis-routing.gen.yaml" + DisableAutomtlsEdgeApisFileName = "disable-automtls-edge-apis-routing.gen.yaml" + UpstreamSslConfigEdgeApisFileName = "upstream-ssl-config-edge-apis.gen.yaml" + UpstreamSslConfigAndDisableAutomtlsFileName = "sslconfig-and-disable-automtls-edge-apis-routing.gen.yaml" httpbinSvc = &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "httpbin", Namespace: "httpbin"}} - // Edge API resources for no sslConfig on Upstream - GetGlooGatewayEdgeResources = func(installNamespace string) []client.Object { + getGlooGatewayEdgeResourceFilmeName = func(config UpstreamConfigOpts) string { + if config.SetSslConfig && config.DisableIstioAutoMtls { + return UpstreamSslConfigAndDisableAutomtlsFileName + } else if config.SetSslConfig { + return UpstreamSslConfigEdgeApisFileName + } else if config.DisableIstioAutoMtls { + return DisableAutomtlsEdgeApisFileName + } else { + return EdgeApisRoutingFileName + } + } + + // GetGlooGatewayEdgeResources defines the Edge API resources based on the UpstreamConfigOpts and the file name of the generated manifest + GetGlooGatewayEdgeResources = func(installNamespace string, config UpstreamConfigOpts) []client.Object { + var sslConfig *ssl.UpstreamSslConfig + if config.SetSslConfig { + /* + This should match the basic istio integration sslConfig: + sslConfig: + alpnProtocols: + - istio + sds: + certificatesSecretName: istio_server_cert + clusterName: gateway_proxy_sds + targetUri: 127.0.0.1:8234 + validationContextName: istio_validation_context + */ + sslConfig = &ssl.UpstreamSslConfig{ + AlpnProtocols: []string{"istio"}, + SslSecrets: &ssl.UpstreamSslConfig_Sds{ + Sds: &ssl.SDSConfig{ + CertificatesSecretName: constants.IstioCertSecret, + SdsBuilder: &ssl.SDSConfig_ClusterName{ + ClusterName: constants.SdsClusterName, + }, + TargetUri: constants.SdsTargetURI, + ValidationContextName: constants.IstioValidationContext, + }, + }, + } + } + httpbinUpstream := &soloapis_gloov1.Upstream{ TypeMeta: metav1.TypeMeta{ Kind: gloov1.UpstreamGVK.Kind, @@ -31,6 +83,7 @@ var ( Namespace: installNamespace, }, Spec: soloapis_gloov1.UpstreamSpec{ + DisableIstioAutoMtls: &wrappers.BoolValue{Value: config.DisableIstioAutoMtls}, UpstreamType: &soloapis_gloov1.UpstreamSpec_Kube{ Kube: &soloapis_kubernetes.UpstreamSpec{ Selector: map[string]string{ @@ -41,6 +94,7 @@ var ( ServicePort: 8000, }, }, + SslConfig: sslConfig, }, } @@ -85,6 +139,7 @@ var ( var resources []client.Object resources = append(resources, headlessVs, httpbinUpstream) + return resources } ) diff --git a/test/kubernetes/e2e/features/istio/inputs/permissive-peer-auth.yaml b/test/kubernetes/e2e/features/istio/testdata/disable-peer-auth.yaml similarity index 77% rename from test/kubernetes/e2e/features/istio/inputs/permissive-peer-auth.yaml rename to test/kubernetes/e2e/features/istio/testdata/disable-peer-auth.yaml index 1980ad89c54..5933f7bbdb2 100644 --- a/test/kubernetes/e2e/features/istio/inputs/permissive-peer-auth.yaml +++ b/test/kubernetes/e2e/features/istio/testdata/disable-peer-auth.yaml @@ -1,8 +1,8 @@ apiVersion: "security.istio.io/v1beta1" kind: "PeerAuthentication" metadata: - name: "test" + name: "disable" namespace: "istio-system" spec: mtls: - mode: PERMISSIVE \ No newline at end of file + mode: DISABLE \ No newline at end of file diff --git a/test/kubernetes/e2e/features/istio/inputs/k8srouting.yaml b/test/kubernetes/e2e/features/istio/testdata/k8s-routing-svc.yaml similarity index 100% rename from test/kubernetes/e2e/features/istio/inputs/k8srouting.yaml rename to test/kubernetes/e2e/features/istio/testdata/k8s-routing-svc.yaml diff --git a/test/kubernetes/e2e/features/istio/testdata/k8s-routing-upstream.yaml b/test/kubernetes/e2e/features/istio/testdata/k8s-routing-upstream.yaml new file mode 100644 index 00000000000..50f25716661 --- /dev/null +++ b/test/kubernetes/e2e/features/istio/testdata/k8s-routing-upstream.yaml @@ -0,0 +1,46 @@ +kind: Gateway +apiVersion: gateway.networking.k8s.io/v1 +metadata: + name: gw +spec: + gatewayClassName: gloo-gateway + listeners: + - protocol: HTTP + port: 8080 + name: http + allowedRoutes: + namespaces: + from: All +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: httpbin-route + namespace: httpbin +spec: + parentRefs: + - name: gw + namespace: default + hostnames: + - "httpbin" + rules: + - backendRefs: + - name: httpbin-upstream + namespace: httpbin + port: 8000 + kind: Upstream + group: gloo.solo.io +--- +apiVersion: gloo.solo.io/v1 +kind: Upstream +metadata: + name: httpbin-upstream + namespace: httpbin +spec: + disableIstioAutoMtls: true + kube: + selector: + app: httpbin + serviceName: httpbin + serviceNamespace: httpbin + servicePort: 8000 diff --git a/test/kubernetes/e2e/features/istio/testdata/permissive-peer-auth.yaml b/test/kubernetes/e2e/features/istio/testdata/permissive-peer-auth.yaml new file mode 100644 index 00000000000..066c8729390 --- /dev/null +++ b/test/kubernetes/e2e/features/istio/testdata/permissive-peer-auth.yaml @@ -0,0 +1,8 @@ +apiVersion: "security.istio.io/v1beta1" +kind: "PeerAuthentication" +metadata: + name: "permissive" + namespace: "istio-system" +spec: + mtls: + mode: PERMISSIVE \ No newline at end of file diff --git a/test/kubernetes/e2e/features/istio/inputs/setup.yaml b/test/kubernetes/e2e/features/istio/testdata/setup.yaml similarity index 100% rename from test/kubernetes/e2e/features/istio/inputs/setup.yaml rename to test/kubernetes/e2e/features/istio/testdata/setup.yaml diff --git a/test/kubernetes/e2e/features/istio/inputs/strict-peer-auth.yaml b/test/kubernetes/e2e/features/istio/testdata/strict-peer-auth.yaml similarity index 78% rename from test/kubernetes/e2e/features/istio/inputs/strict-peer-auth.yaml rename to test/kubernetes/e2e/features/istio/testdata/strict-peer-auth.yaml index c5119f5c687..3065fe83c99 100644 --- a/test/kubernetes/e2e/features/istio/inputs/strict-peer-auth.yaml +++ b/test/kubernetes/e2e/features/istio/testdata/strict-peer-auth.yaml @@ -1,7 +1,7 @@ apiVersion: "security.istio.io/v1beta1" kind: "PeerAuthentication" metadata: - name: "test" + name: "strict" namespace: "istio-system" spec: mtls: diff --git a/test/kubernetes/e2e/features/istio/types.go b/test/kubernetes/e2e/features/istio/types.go index 24c80e69d90..bdd59ff56a7 100644 --- a/test/kubernetes/e2e/features/istio/types.go +++ b/test/kubernetes/e2e/features/istio/types.go @@ -15,10 +15,14 @@ import ( ) var ( - setupManifest = filepath.Join(util.MustGetThisDir(), "inputs/setup.yaml") - strictPeerAuthManifest = filepath.Join(util.MustGetThisDir(), "inputs/strict-peer-auth.yaml") - permissivePeerAuthManifest = filepath.Join(util.MustGetThisDir(), "inputs/permissive-peer-auth.yaml") - k8sRoutingManifest = filepath.Join(util.MustGetThisDir(), "inputs/k8srouting.yaml") + setupManifest = filepath.Join(util.MustGetThisDir(), "testdata", "setup.yaml") + + strictPeerAuthManifest = filepath.Join(util.MustGetThisDir(), "testdata", "strict-peer-auth.yaml") + permissivePeerAuthManifest = filepath.Join(util.MustGetThisDir(), "testdata", "permissive-peer-auth.yaml") + disablePeerAuthManifest = filepath.Join(util.MustGetThisDir(), "testdata", "disable-peer-auth.yaml") + + k8sRoutingSvcManifest = filepath.Join(util.MustGetThisDir(), "testdata", "k8s-routing-svc.yaml") + k8sRoutingUpstreamManifest = filepath.Join(util.MustGetThisDir(), "testdata", "k8s-routing-upstream.yaml") // When we apply the fault injection manifest files, we expect resources to be created with this metadata glooProxyObjectMeta = metav1.ObjectMeta{ @@ -55,6 +59,6 @@ var ( expectedServiceUnavailableResponse = &testmatchers.HttpResponse{ StatusCode: http.StatusServiceUnavailable, - Body: gomega.ContainSubstring("upstream connect error or disconnect/reset before headers. reset reason: connection termination"), + Body: gomega.ContainSubstring("upstream connect error or disconnect/reset before headers"), } ) diff --git a/test/kubernetes/e2e/features/route_options/suite.go b/test/kubernetes/e2e/features/route_options/suite.go index 73913d26bdd..9d090e06c83 100644 --- a/test/kubernetes/e2e/features/route_options/suite.go +++ b/test/kubernetes/e2e/features/route_options/suite.go @@ -41,15 +41,27 @@ func NewTestingSuite(ctx context.Context, testInst *e2e.TestInstallation) suite. } func (s *testingSuite) SetupSuite() { + // Check that the common setup manifest is applied + err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, setupManifest) + s.NoError(err, "can apply "+setupManifest) + s.testInstallation.Assertions.EventuallyObjectsExist(s.ctx, proxyService, proxyDeployment, exampleSvc, nginxPod) + // Check that test resources are running + s.testInstallation.Assertions.EventuallyPodsRunning(s.ctx, nginxPod.ObjectMeta.GetNamespace(), metav1.ListOptions{ + LabelSelector: "app.kubernetes.io/name=nginx", + }) + s.testInstallation.Assertions.EventuallyPodsRunning(s.ctx, proxyDeployment.ObjectMeta.GetNamespace(), metav1.ListOptions{ + LabelSelector: "app.kubernetes.io/name=gloo-proxy-gw", + }) + // We include tests with manual setup here because the cleanup is still automated via AfterTest s.manifests = map[string][]string{ - "TestConfigureRouteOptionsWithTargetRef": {setupManifest, httproute1Manifest, basicRtoTargetRefManifest}, - "TestConfigureRouteOptionsWithFilterExtension": {setupManifest, basicRtoManifest, httproute1ExtensionManifest}, - "TestConfigureInvalidRouteOptionsWithTargetRef": {setupManifest, httproute1Manifest, badRtoTargetRefManifest}, - "TestConfigureInvalidRouteOptionsWithFilterExtension": {setupManifest, httproute1BadExtensionManifest, badRtoManifest}, - "TestConfigureRouteOptionsWithMultipleTargetRefManualSetup": {setupManifest, httproute1Manifest, basicRtoTargetRefManifest, extraRtoTargetRefManifest}, - "TestConfigureRouteOptionsWithMultipleFilterExtensionManualSetup": {setupManifest, httproute1MultipleExtensionsManifest, basicRtoManifest, extraRtoManifest}, - "TestConfigureRouteOptionsWithTargetRefAndFilterExtension": {setupManifest, httproute1ExtensionManifest, basicRtoManifest, extraRtoTargetRefManifest}, + "TestConfigureRouteOptionsWithTargetRef": {httproute1Manifest, basicRtoTargetRefManifest}, + "TestConfigureRouteOptionsWithFilterExtension": {basicRtoManifest, httproute1ExtensionManifest}, + "TestConfigureInvalidRouteOptionsWithTargetRef": {httproute1Manifest, badRtoTargetRefManifest}, + "TestConfigureInvalidRouteOptionsWithFilterExtension": {httproute1BadExtensionManifest, badRtoManifest}, + "TestConfigureRouteOptionsWithMultipleTargetRefManualSetup": {httproute1Manifest, basicRtoTargetRefManifest, extraRtoTargetRefManifest}, + "TestConfigureRouteOptionsWithMultipleFilterExtensionManualSetup": {httproute1MultipleExtensionsManifest, basicRtoManifest, extraRtoManifest}, + "TestConfigureRouteOptionsWithTargetRefAndFilterExtension": {httproute1ExtensionManifest, basicRtoManifest, extraRtoTargetRefManifest}, } } @@ -74,7 +86,7 @@ func (s *testingSuite) BeforeTest(suiteName, testName string) { func (s *testingSuite) AfterTest(suiteName, testName string) { manifests, ok := s.manifests[testName] if !ok { - s.Fail("no manifests found for " + testName) + s.FailNow("no manifests found for " + testName) } for _, manifest := range manifests { diff --git a/test/kubernetes/e2e/features/route_options/types.go b/test/kubernetes/e2e/features/route_options/types.go index ec31e5523e9..0c1bffe8252 100644 --- a/test/kubernetes/e2e/features/route_options/types.go +++ b/test/kubernetes/e2e/features/route_options/types.go @@ -8,6 +8,7 @@ import ( "github.com/onsi/gomega/gstruct" "github.com/solo-io/gloo/test/gomega/matchers" "github.com/solo-io/skv2/codegen/util" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -30,6 +31,15 @@ var ( proxyService = &corev1.Service{ ObjectMeta: objectMetaInDefault("gloo-proxy-gw"), } + proxyDeployment = &appsv1.Deployment{ + ObjectMeta: objectMetaInDefault("gloo-proxy-gw"), + } + nginxPod = &corev1.Pod{ + ObjectMeta: objectMetaInDefault("nginx"), + } + exampleSvc = &corev1.Service{ + ObjectMeta: objectMetaInDefault("example-svc"), + } // RouteOption resources to be created basicRtoMeta = objectMetaInDefault("basic-rto") diff --git a/test/kubernetes/e2e/features/virtualhost_options/suite.go b/test/kubernetes/e2e/features/virtualhost_options/suite.go index b042325e71d..be7e9b271e0 100644 --- a/test/kubernetes/e2e/features/virtualhost_options/suite.go +++ b/test/kubernetes/e2e/features/virtualhost_options/suite.go @@ -40,12 +40,24 @@ func NewTestingSuite(ctx context.Context, testInst *e2e.TestInstallation) suite. } func (s *testingSuite) SetupSuite() { + // Check that the common setup manifest is applied + err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, setupManifest) + s.NoError(err, "can apply "+setupManifest) + s.testInstallation.Assertions.EventuallyObjectsExist(s.ctx, proxyService, proxyDeployment, exampleSvc, nginxPod) + // Check that test resources are running + s.testInstallation.Assertions.EventuallyPodsRunning(s.ctx, nginxPod.ObjectMeta.GetNamespace(), metav1.ListOptions{ + LabelSelector: "app.kubernetes.io/name=nginx", + }) + s.testInstallation.Assertions.EventuallyPodsRunning(s.ctx, proxyDeployment.ObjectMeta.GetNamespace(), metav1.ListOptions{ + LabelSelector: "app.kubernetes.io/name=gloo-proxy-gw", + }) + // We include tests with manual setup here because the cleanup is still automated via AfterTest s.manifests = map[string][]string{ - "TestConfigureVirtualHostOptions": {setupManifest, basicVhOManifest}, - "TestConfigureInvalidVirtualHostOptions": {setupManifest, basicVhOManifest, badVhOManifest}, - "TestConfigureVirtualHostOptionsWithSectionNameManualSetup": {setupManifest, basicVhOManifest, extraVhOManifest, sectionNameVhOManifest}, - "TestMultipleVirtualHostOptionsManualSetup": {setupManifest, basicVhOManifest, extraVhOManifest}, + "TestConfigureVirtualHostOptions": {basicVhOManifest}, + "TestConfigureInvalidVirtualHostOptions": {basicVhOManifest, badVhOManifest}, + "TestConfigureVirtualHostOptionsWithSectionNameManualSetup": {basicVhOManifest, extraVhOManifest, sectionNameVhOManifest}, + "TestMultipleVirtualHostOptionsManualSetup": {basicVhOManifest, extraVhOManifest}, } } @@ -70,7 +82,7 @@ func (s *testingSuite) BeforeTest(suiteName, testName string) { func (s *testingSuite) AfterTest(suiteName, testName string) { manifests, ok := s.manifests[testName] if !ok { - s.Fail("no manifests found for " + testName) + s.FailNow("no manifests found for " + testName) } for _, manifest := range manifests { diff --git a/test/kubernetes/e2e/features/virtualhost_options/types.go b/test/kubernetes/e2e/features/virtualhost_options/types.go index 9cb026c5061..e836e607904 100644 --- a/test/kubernetes/e2e/features/virtualhost_options/types.go +++ b/test/kubernetes/e2e/features/virtualhost_options/types.go @@ -8,6 +8,7 @@ import ( "github.com/onsi/gomega/gstruct" "github.com/solo-io/gloo/test/gomega/matchers" "github.com/solo-io/skv2/codegen/util" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -24,7 +25,25 @@ var ( Name: "gloo-proxy-gw", Namespace: "default", } - proxyService = &corev1.Service{ObjectMeta: glooProxyObjectMeta} + proxyService = &corev1.Service{ObjectMeta: glooProxyObjectMeta} + proxyDeployment = &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gloo-proxy-gw", + Namespace: "default", + }, + } + nginxPod = &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nginx", + Namespace: "default", + }, + } + exampleSvc = &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "example-svc", + Namespace: "default", + }, + } // VirtualHostOption resource to be created basicVirtualHostOptionMeta = metav1.ObjectMeta{ diff --git a/test/kubernetes/e2e/test.go b/test/kubernetes/e2e/test.go index b5f97fd035a..d3786ba74e9 100644 --- a/test/kubernetes/e2e/test.go +++ b/test/kubernetes/e2e/test.go @@ -81,8 +81,9 @@ func CreateTestInstallationForCluster( // GeneratedFiles contains the unique location where files generated during the execution // of tests against this installation will be stored - // By creating a unique location, per TestInstallation, we guarantee isolation between TestInstallation - GeneratedFiles: MustGeneratedFiles(glooGatewayContext.InstallNamespace), + // By creating a unique location, per TestInstallation and per ClusterId we guarantee isolation + // between TestInstallation outputs per CI run + GeneratedFiles: MustGeneratedFiles(glooGatewayContext.InstallNamespace, clusterContext.ClusterId), } runtime.SetFinalizer(installation, func(i *TestInstallation) { i.finalize() }) return installation @@ -196,13 +197,14 @@ type GeneratedFiles struct { } // MustGeneratedFiles returns GeneratedFiles, or panics if there was an error generating the directories -func MustGeneratedFiles(tmpDirId string) GeneratedFiles { +func MustGeneratedFiles(tmpDirId, clusterId string) GeneratedFiles { tmpDir, err := os.MkdirTemp("", tmpDirId) if err != nil { panic(err) } - failureDir := filepath.Join(testruntime.PathToBugReport(), tmpDirId) + // output path is in the format of bug_report/cluster_id/tmp_dir_id + failureDir := filepath.Join(testruntime.PathToBugReport(), clusterId, tmpDirId) err = os.MkdirAll(failureDir, os.ModePerm) if err != nil { panic(err) diff --git a/test/kubernetes/e2e/tests/automtls_istio_edge_api_test.go b/test/kubernetes/e2e/tests/automtls_istio_edge_api_test.go index 9db122ca885..88471c41241 100644 --- a/test/kubernetes/e2e/tests/automtls_istio_edge_api_test.go +++ b/test/kubernetes/e2e/tests/automtls_istio_edge_api_test.go @@ -40,6 +40,9 @@ func TestAutomtlsIstioEdgeApisGateway(t *testing.T) { t.Cleanup(func() { if t.Failed() { testInstallation.PreFailHandler(ctx) + + // Generate istioctl bug report + testInstallation.CreateIstioBugReport(ctx) } testInstallation.UninstallGlooGateway(ctx, func(ctx context.Context) error { @@ -49,14 +52,14 @@ func TestAutomtlsIstioEdgeApisGateway(t *testing.T) { // Uninstall Istio err = testInstallation.UninstallIstio() if err != nil { - t.Fatalf("failed to uninstall istio: %v", err) + t.Fatalf("failed to uninstall: %v\n", err) } }) // Install Istio before Gloo Gateway to make sure istiod is present before istio-proxy err = testInstallation.InstallMinimalIstio(ctx) if err != nil { - t.Fatalf("failed to install istio: %v", err) + t.Fatalf("failed to install: %v\n", err) } // Install Gloo Gateway with only Gloo Edge Gateway APIs enabled diff --git a/test/kubernetes/e2e/tests/automtls_istio_test.go b/test/kubernetes/e2e/tests/automtls_istio_test.go index 6e00226c1cf..0f4916c901e 100644 --- a/test/kubernetes/e2e/tests/automtls_istio_test.go +++ b/test/kubernetes/e2e/tests/automtls_istio_test.go @@ -2,6 +2,7 @@ package tests_test import ( "context" + "log" "path/filepath" "testing" "time" @@ -30,7 +31,8 @@ func TestK8sGatewayIstioAutoMtls(t *testing.T) { testHelper := e2e.MustTestHelper(ctx, testInstallation) err := testInstallation.AddIstioctl(ctx) if err != nil { - t.Fatalf("failed to get istioctl: %v", err) + log.Printf("failed to install: %v\n", err) + t.Fail() } // We register the cleanup function _before_ we actually perform the installation. @@ -50,14 +52,16 @@ func TestK8sGatewayIstioAutoMtls(t *testing.T) { // Uninstall Istio err = testInstallation.UninstallIstio() if err != nil { - t.Fatalf("failed to uninstall istio: %v", err) + log.Printf("failed to uninstall: %v\n", err) + t.Fail() } }) // Install Istio before Gloo Gateway to make sure istiod is present before istio-proxy err = testInstallation.InstallMinimalIstio(ctx) if err != nil { - t.Fatalf("failed to install istio: %v", err) + log.Printf("failed to install: %v\n", err) + t.Fail() } // Install Gloo Gateway diff --git a/test/kubernetes/e2e/tests/glooctl_istio_inject_test.go b/test/kubernetes/e2e/tests/glooctl_istio_inject_test.go index f2abbd4bd81..7ace0f38417 100644 --- a/test/kubernetes/e2e/tests/glooctl_istio_inject_test.go +++ b/test/kubernetes/e2e/tests/glooctl_istio_inject_test.go @@ -31,7 +31,7 @@ func TestGlooctlIstioInjectEdgeApiGateway(t *testing.T) { testHelper := e2e.MustTestHelper(ctx, testInstallation) err := testInstallation.AddIstioctl(ctx) if err != nil { - t.Fatalf("failed to get istioctl: %v", err) + t.Fatalf("failed to get istioctl: %v\n", err) } // We register the cleanup function _before_ we actually perform the installation. @@ -39,6 +39,9 @@ func TestGlooctlIstioInjectEdgeApiGateway(t *testing.T) { t.Cleanup(func() { if t.Failed() { testInstallation.PreFailHandler(ctx) + + // Generate istioctl bug report + testInstallation.CreateIstioBugReport(ctx) } testInstallation.UninstallGlooGateway(ctx, func(ctx context.Context) error { @@ -48,14 +51,14 @@ func TestGlooctlIstioInjectEdgeApiGateway(t *testing.T) { // Uninstall Istio err = testInstallation.UninstallIstio() if err != nil { - t.Fatalf("failed to uninstall istio: %v", err) + t.Fatalf("failed to uninstall: %v\n", err) } }) // Install Istio before Gloo Gateway to make sure istiod is present before istio-proxy err = testInstallation.InstallMinimalIstio(ctx) if err != nil { - t.Fatalf("failed to install istio: %v", err) + t.Fatalf("failed to install: %v\n", err) } // Install Gloo Gateway with only Gloo Edge Gateway APIs enabled diff --git a/test/kubernetes/e2e/tests/istio_edge_api_test.go b/test/kubernetes/e2e/tests/istio_edge_api_test.go index 3641a9ae66c..4e06196ae7b 100644 --- a/test/kubernetes/e2e/tests/istio_edge_api_test.go +++ b/test/kubernetes/e2e/tests/istio_edge_api_test.go @@ -2,6 +2,7 @@ package tests_test import ( "context" + "log" "path/filepath" "testing" "time" @@ -33,7 +34,8 @@ func TestIstioEdgeApiGateway(t *testing.T) { err := testInstallation.AddIstioctl(ctx) if err != nil { - t.Fatalf("failed to get istioctl: %v", err) + log.Printf("failed to add istioctl: %v\n", err) + t.Fail() } // We register the cleanup function _before_ we actually perform the installation. @@ -41,6 +43,9 @@ func TestIstioEdgeApiGateway(t *testing.T) { t.Cleanup(func() { if t.Failed() { testInstallation.PreFailHandler(ctx) + + // Generate istioctl bug report + testInstallation.CreateIstioBugReport(ctx) } testInstallation.UninstallGlooGateway(ctx, func(ctx context.Context) error { @@ -50,14 +55,16 @@ func TestIstioEdgeApiGateway(t *testing.T) { // Uninstall Istio err = testInstallation.UninstallIstio() if err != nil { - t.Fatalf("failed to uninstall istio: %v", err) + log.Printf("failed to uninstall: %v\n", err) + t.Fail() } }) // Install Istio before Gloo Gateway to make sure istiod is present before istio-proxy err = testInstallation.InstallMinimalIstio(ctx) if err != nil { - t.Fatalf("failed to install istio: %v", err) + log.Printf("failed to install: %v\n", err) + t.Fail() } // Install Gloo Gateway with only Edge APIs enabled diff --git a/test/kubernetes/e2e/tests/istio_test.go b/test/kubernetes/e2e/tests/istio_test.go index 10e44555dbf..6eaf88fcf6f 100644 --- a/test/kubernetes/e2e/tests/istio_test.go +++ b/test/kubernetes/e2e/tests/istio_test.go @@ -50,14 +50,14 @@ func TestK8sGatewayIstio(t *testing.T) { // Uninstall Istio err = testInstallation.UninstallIstio() if err != nil { - t.Fatalf("failed to uninstall istio: %v", err) + t.Fatalf("failed to uninstall: %v\n", err) } }) // Install Istio before Gloo Gateway to make sure istiod is present before istio-proxy err = testInstallation.InstallMinimalIstio(ctx) if err != nil { - t.Fatalf("failed to install istio: %v", err) + t.Fatalf("failed to install: %v", err) } // Install Gloo Gateway diff --git a/test/kubernetes/testutils/assertions/curl.go b/test/kubernetes/testutils/assertions/curl.go index 499f39c27f8..a599a29230f 100644 --- a/test/kubernetes/testutils/assertions/curl.go +++ b/test/kubernetes/testutils/assertions/curl.go @@ -44,7 +44,7 @@ func (p *Provider) AssertEventualCurlResponse( WithTimeout(currentTimeout). WithPolling(pollingInterval). WithContext(ctx). - Should(Succeed()) + Should(Succeed(), "failed to get expected response") } // AssertEventuallyConsistentCurlResponse asserts that the response from a curl command diff --git a/test/kubernetes/testutils/assertions/objects.go b/test/kubernetes/testutils/assertions/objects.go index 13ceeed779c..912522854df 100644 --- a/test/kubernetes/testutils/assertions/objects.go +++ b/test/kubernetes/testutils/assertions/objects.go @@ -19,9 +19,9 @@ func (p *Provider) EventuallyObjectsExist(ctx context.Context, objects ...client innerG.Expect(err).NotTo(HaveOccurred(), "object %s %s should be available in cluster", o.GetObjectKind().GroupVersionKind().String(), client.ObjectKeyFromObject(o).String()) }). WithContext(ctx). - WithTimeout(time.Second * 20). - WithPolling(time.Millisecond * 200). - Should(Succeed()) + WithTimeout(time.Second*20). + WithPolling(time.Millisecond*200). + Should(Succeed(), fmt.Sprintf("object %s %s should be available in cluster", o.GetObjectKind().GroupVersionKind().String(), client.ObjectKeyFromObject(o).String())) } } @@ -32,9 +32,9 @@ func (p *Provider) EventuallyObjectsNotExist(ctx context.Context, objects ...cli innerG.Expect(apierrors.IsNotFound(err)).To(BeTrue(), "object %s %s should not be found in cluster", o.GetObjectKind().GroupVersionKind().String(), client.ObjectKeyFromObject(o).String()) }). WithContext(ctx). - WithTimeout(time.Second * 20). - WithPolling(time.Millisecond * 200). - Should(Succeed()) + WithTimeout(time.Second*20). + WithPolling(time.Millisecond*200). + Should(Succeed(), fmt.Sprintf("object %s %s should not be found in cluster", o.GetObjectKind().GroupVersionKind().String(), client.ObjectKeyFromObject(o).String())) } } diff --git a/test/kubernetes/testutils/assertions/pods.go b/test/kubernetes/testutils/assertions/pods.go index b34d426ebcd..b474590bfd4 100644 --- a/test/kubernetes/testutils/assertions/pods.go +++ b/test/kubernetes/testutils/assertions/pods.go @@ -6,20 +6,41 @@ import ( "github.com/onsi/gomega" "github.com/onsi/gomega/types" + "github.com/solo-io/gloo/test/gomega/matchers" + "github.com/solo-io/gloo/test/kube2e/helper" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +// EventuallyPodsRunning asserts that the pod(s) are in the ready state +func (p *Provider) EventuallyPodsRunning( + ctx context.Context, + podNamespace string, + listOpt metav1.ListOptions, + timeout ...time.Duration, +) { + p.EventuallyPodsMatches(ctx, podNamespace, listOpt, matchers.PodMatches(matchers.ExpectedPod{Status: corev1.PodRunning}), timeout...) +} + // EventuallyPodsMatches asserts that the pod(s) in the given namespace matches the provided matcher -func (p *Provider) EventuallyPodsMatches(ctx context.Context, podNamespace string, listOpt metav1.ListOptions, matcher types.GomegaMatcher) { +func (p *Provider) EventuallyPodsMatches( + ctx context.Context, + podNamespace string, + listOpt metav1.ListOptions, + matcher types.GomegaMatcher, + timeout ...time.Duration, +) { + currentTimeout, pollingInterval := helper.GetTimeouts(timeout...) + p.Gomega.Eventually(func(g gomega.Gomega) { - proxyPods, err := p.clusterContext.Clientset.CoreV1().Pods(podNamespace).List(ctx, listOpt) + pods, err := p.clusterContext.Clientset.CoreV1().Pods(podNamespace).List(ctx, listOpt) g.Expect(err).NotTo(gomega.HaveOccurred(), "Failed to list pods") - g.Expect(proxyPods.Items).NotTo(gomega.BeEmpty(), "No pods found") - for _, pod := range proxyPods.Items { + g.Expect(pods.Items).NotTo(gomega.BeEmpty(), "No pods found") + for _, pod := range pods.Items { g.Expect(pod).To(matcher) } }). - WithTimeout(time.Second*60). - WithPolling(time.Second*5). + WithTimeout(currentTimeout). + WithPolling(pollingInterval). Should(gomega.Succeed(), "Failed to match pod") } diff --git a/test/kubernetes/testutils/cluster/context.go b/test/kubernetes/testutils/cluster/context.go index 5c916e1995f..43055eeb117 100644 --- a/test/kubernetes/testutils/cluster/context.go +++ b/test/kubernetes/testutils/cluster/context.go @@ -28,4 +28,7 @@ type Context struct { // A set of clients for interacting with the Kubernetes Cluster Clientset *kubernetes.Clientset + + // Cluster id is a unique identifier for the cluster running in CI + ClusterId string } diff --git a/test/kubernetes/testutils/cluster/kind.go b/test/kubernetes/testutils/cluster/kind.go index fd112e241aa..35bd670923a 100644 --- a/test/kubernetes/testutils/cluster/kind.go +++ b/test/kubernetes/testutils/cluster/kind.go @@ -4,6 +4,7 @@ import ( "fmt" "os" + "github.com/solo-io/gloo/test/testutils" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes" @@ -56,5 +57,7 @@ func MustKindContextWithScheme(clusterName string, scheme *runtime.Scheme) *Cont Cli: kubectl.NewCli().WithKubeContext(kubeCtx).WithReceiver(os.Stdout), Client: clt, Clientset: clientset, + // Get cluster id if it is set in the env to avoid conflicts when uploading artifacts on failed workflow runs + ClusterId: os.Getenv(testutils.ClusterId), } } diff --git a/test/testutils/env.go b/test/testutils/env.go index e50f4a5db3e..b0c30fd5aea 100644 --- a/test/testutils/env.go +++ b/test/testutils/env.go @@ -80,6 +80,9 @@ const ( // ClusterName is the name of the cluster used for e2e tests ClusterName = "CLUSTER_NAME" + + // ClusterId is the name of the cluster id used for e2e tests run in ci + ClusterId = "CLUSTER_ID" ) // ShouldTearDown returns true if any assets that were created before a test (for example Gloo being installed)