From 16383c8f917868f71450943b2c8f79a7b646594e Mon Sep 17 00:00:00 2001 From: Jonathan Knight Date: Mon, 25 Sep 2023 15:25:52 +0300 Subject: [PATCH] Make Istio tests deploy Istio in strict mode (#620) --- .github/workflows/build.yaml | 8 +- .github/workflows/coherence-matrix.yaml | 12 +- .github/workflows/compatibility-tests.yaml | 8 +- .github/workflows/doc-check.yaml | 8 +- .github/workflows/istio-tests.yaml | 8 +- .github/workflows/k8s-matrix.yaml | 16 +-- .github/workflows/minikube-matrix.yaml | 20 +-- .github/workflows/prometheus-tests.yaml | 8 +- .github/workflows/release.yml | 8 +- .github/workflows/tanzu-tests.yaml | 8 +- .github/workflows/trivy.yaml | 8 +- Makefile | 19 ++- api/v1/coherence_types.go | 33 +++-- api/v1/common_test.go | 8 ++ docs/installation/01_installation.adoc | 15 +- examples/400_Istio/README.adoc | 158 +++++++++++++++++---- hack/istio-operator.yaml | 16 +++ hack/istio-strict.yaml | 9 ++ test/e2e/clients/storage.yaml | 5 + 19 files changed, 274 insertions(+), 101 deletions(-) create mode 100644 hack/istio-operator.yaml create mode 100644 hack/istio-strict.yaml diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index ccb2432ec..5c7e07159 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -63,15 +63,15 @@ jobs: df -h - name: Set up JDK - uses: actions/setup-java@v3 + uses: oracle-actions/setup-java@v1 with: - distribution: 'zulu' - java-version: '11' + website: oracle.com + release: 21 - name: Set up Go uses: actions/setup-go@v3 with: - go-version: 1.19.x + go-version: 1.20.x - name: Cache Go Modules uses: actions/cache@v3 diff --git a/.github/workflows/coherence-matrix.yaml b/.github/workflows/coherence-matrix.yaml index ef1d47391..b4e158741 100644 --- a/.github/workflows/coherence-matrix.yaml +++ b/.github/workflows/coherence-matrix.yaml @@ -70,8 +70,8 @@ jobs: baseImage: "gcr.io/distroless/java17-debian11" - matrixName: "22.06" - coherenceVersion: "22.06.4" - coherenceImage: "ghcr.io/oracle/coherence-ce:22.06.4" + coherenceVersion: "22.06.5" + coherenceImage: "ghcr.io/oracle/coherence-ce:22.06.5" javaVersion: 11 baseImage: "gcr.io/distroless/java11-debian11" @@ -139,15 +139,15 @@ jobs: df -h - name: Set up JDK - uses: actions/setup-java@v3 + uses: oracle-actions/setup-java@v1 with: - distribution: 'zulu' - java-version: '17' + website: oracle.com + release: 21 - name: Set up Go uses: actions/setup-go@v3 with: - go-version: 1.19.x + go-version: 1.20.x - name: Cache Go Modules uses: actions/cache@v3 diff --git a/.github/workflows/compatibility-tests.yaml b/.github/workflows/compatibility-tests.yaml index 31e49a244..5d9eb5e3a 100644 --- a/.github/workflows/compatibility-tests.yaml +++ b/.github/workflows/compatibility-tests.yaml @@ -97,15 +97,15 @@ jobs: df -h - name: Set up JDK - uses: actions/setup-java@v3 + uses: oracle-actions/setup-java@v1 with: - distribution: 'zulu' - java-version: '11' + website: oracle.com + release: 21 - name: Set up Go uses: actions/setup-go@v3 with: - go-version: 1.19.x + go-version: 1.20.x - name: Cache Go Modules uses: actions/cache@v3 diff --git a/.github/workflows/doc-check.yaml b/.github/workflows/doc-check.yaml index 727f4a6fa..108edff29 100644 --- a/.github/workflows/doc-check.yaml +++ b/.github/workflows/doc-check.yaml @@ -31,15 +31,15 @@ jobs: fetch-depth: 0 - name: Set up JDK - uses: actions/setup-java@v3 + uses: oracle-actions/setup-java@v1 with: - distribution: 'zulu' - java-version: '11' + website: oracle.com + release: 21 - name: Set up Go uses: actions/setup-go@v3 with: - go-version: 1.19.x + go-version: 1.20.x - name: Cache Go Modules uses: actions/cache@v3 diff --git a/.github/workflows/istio-tests.yaml b/.github/workflows/istio-tests.yaml index f4ac5ab66..89f6e3fec 100644 --- a/.github/workflows/istio-tests.yaml +++ b/.github/workflows/istio-tests.yaml @@ -69,15 +69,15 @@ jobs: df -h - name: Set up JDK - uses: actions/setup-java@v3 + uses: oracle-actions/setup-java@v1 with: - distribution: 'zulu' - java-version: '11' + website: oracle.com + release: 21 - name: Set up Go uses: actions/setup-go@v3 with: - go-version: 1.19.x + go-version: 1.20.x - name: Cache Go Modules uses: actions/cache@v3 diff --git a/.github/workflows/k8s-matrix.yaml b/.github/workflows/k8s-matrix.yaml index c98a129fa..5e0b62258 100644 --- a/.github/workflows/k8s-matrix.yaml +++ b/.github/workflows/k8s-matrix.yaml @@ -49,19 +49,19 @@ jobs: - v1.21 include: - matrixName: v1.27 - k8s: kindest/node:v1.27.1@sha256:9915f5629ef4d29f35b478e819249e89cfaffcbfeebda4324e5c01d53d937b09 + k8s: kindest/node:v1.27.3@sha256:3966ac761ae0136263ffdb6cfd4db23ef8a83cba8a463690e98317add2c9ba72 kindCommand: kind-calico runNetTests: true - matrixName: v1.26 - k8s: kindest/node:v1.26.3@sha256:61b92f38dff6ccc29969e7aa154d34e38b89443af1a2c14e6cfbd2df6419c66f + k8s: kindest/node:v1.26.6@sha256:6e2d8b28a5b601defe327b98bd1c2d1930b49e5d8c512e1895099e4504007adb kindCommand: kind-calico runNetTests: true - matrixName: v1.25 - k8s: kindest/node:v1.25.8@sha256:00d3f5314cc35327706776e95b2f8e504198ce59ac545d0200a89e69fce10b7f + k8s: kindest/node:v1.25.11@sha256:227fa11ce74ea76a0474eeefb84cb75d8dad1b08638371ecf0e86259b35be0c8 kindCommand: kind-calico runNetTests: true - matrixName: v1.24 - k8s: kindest/node:v1.24.12@sha256:1e12918b8bc3d4253bc08f640a231bb0d3b2c5a9b28aa3f2ca1aee93e1e8db16 + k8s: kindest/node:v1.24.15@sha256:7db4f8bea3e14b82d12e044e25e34bd53754b7f2b0e9d56df21774e6f66a70ab kindCommand: kind-calico runNetTests: true - matrixName: v1.23 @@ -105,15 +105,15 @@ jobs: df -h - name: Set up JDK - uses: actions/setup-java@v3 + uses: oracle-actions/setup-java@v1 with: - distribution: 'zulu' - java-version: '11' + website: oracle.com + release: 21 - name: Set up Go uses: actions/setup-go@v3 with: - go-version: 1.19.x + go-version: 1.20.x - name: Cache Go Modules uses: actions/cache@v3 diff --git a/.github/workflows/minikube-matrix.yaml b/.github/workflows/minikube-matrix.yaml index b7a644bc1..e76f01ce8 100644 --- a/.github/workflows/minikube-matrix.yaml +++ b/.github/workflows/minikube-matrix.yaml @@ -40,19 +40,19 @@ jobs: fail-fast: false matrix: matrixName: + - v1.28 - v1.27 - v1.26 - v1.25 - - v1.24 include: + - matrixName: v1.28 + k8s: v1.28.2 - matrixName: v1.27 - k8s: v1.27.1 + k8s: v1.27.6 - matrixName: v1.26 - k8s: v1.26.3 + k8s: v1.26.9 - matrixName: v1.25 - k8s: 1.25.8 - - matrixName: v1.24 - k8s: 1.24.12 + k8s: 1.25.14 steps: - uses: actions/checkout@v3 @@ -82,15 +82,15 @@ jobs: df -h - name: Set up JDK - uses: actions/setup-java@v3 + uses: oracle-actions/setup-java@v1 with: - distribution: 'zulu' - java-version: '11' + website: oracle.com + release: 21 - name: Set up Go uses: actions/setup-go@v3 with: - go-version: 1.19.x + go-version: 1.20.x - name: Cache Go Modules uses: actions/cache@v3 diff --git a/.github/workflows/prometheus-tests.yaml b/.github/workflows/prometheus-tests.yaml index df82b0bf3..f5d7ab8ba 100644 --- a/.github/workflows/prometheus-tests.yaml +++ b/.github/workflows/prometheus-tests.yaml @@ -63,15 +63,15 @@ jobs: df -h - name: Set up JDK - uses: actions/setup-java@v3 + uses: oracle-actions/setup-java@v1 with: - distribution: 'zulu' - java-version: '11' + website: oracle.com + release: 21 - name: Set up Go uses: actions/setup-go@v3 with: - go-version: 1.19.x + go-version: 1.20.x - name: Cache Go Modules uses: actions/cache@v3 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1cc362aed..8dd3be2c5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,15 +24,15 @@ jobs: fetch-depth: 0 - name: Set up JDK - uses: actions/setup-java@v3 + uses: oracle-actions/setup-java@v1 with: - distribution: 'zulu' - java-version: '11' + website: oracle.com + release: 21 - name: Set up Go uses: actions/setup-go@v3 with: - go-version: 1.19.x + go-version: 1.20.x - name: Cache Go Modules uses: actions/cache@v3 diff --git a/.github/workflows/tanzu-tests.yaml b/.github/workflows/tanzu-tests.yaml index 3431fac54..f2112d01d 100644 --- a/.github/workflows/tanzu-tests.yaml +++ b/.github/workflows/tanzu-tests.yaml @@ -67,15 +67,15 @@ jobs: df -h - name: Set up JDK - uses: actions/setup-java@v3 + uses: oracle-actions/setup-java@v1 with: - distribution: 'zulu' - java-version: '11' + website: oracle.com + release: 21 - name: Set up Go uses: actions/setup-go@v3 with: - go-version: 1.19.x + go-version: 1.20.x - name: Cache Go Modules uses: actions/cache@v3 diff --git a/.github/workflows/trivy.yaml b/.github/workflows/trivy.yaml index 9a65df32d..b81a0474e 100644 --- a/.github/workflows/trivy.yaml +++ b/.github/workflows/trivy.yaml @@ -53,15 +53,15 @@ jobs: df -h - name: Set up JDK - uses: actions/setup-java@v3 + uses: oracle-actions/setup-java@v1 with: - distribution: 'zulu' - java-version: '11' + website: oracle.com + release: 21 - name: Set up Go uses: actions/setup-go@v3 with: - go-version: 1.19.x + go-version: 1.20.x - name: Cache Go Modules uses: actions/cache@v3 diff --git a/Makefile b/Makefile index 87fb05357..51156e124 100644 --- a/Makefile +++ b/Makefile @@ -332,7 +332,7 @@ TEST_SSL_SECRET := coherence-ssl-secret # ---------------------------------------------------------------------------------------------------------------------- # Prometheus Operator settings (used in integration tests) # ---------------------------------------------------------------------------------------------------------------------- -PROMETHEUS_VERSION ?= v0.10.0 +PROMETHEUS_VERSION ?= v0.13.0 PROMETHEUS_HOME = $(TOOLS_DIRECTORY)/prometheus/$(PROMETHEUS_VERSION) PROMETHEUS_NAMESPACE ?= monitoring PROMETHEUS_ADAPTER_VERSION ?= 2.5.0 @@ -1558,8 +1558,7 @@ create-ssl-secrets: $(BUILD_OUTPUT)/certs ##@ KinD KIND_CLUSTER ?= operator -KIND_IMAGE ?= "kindest/node:v1.24.7@sha256:577c630ce8e509131eab1aea12c022190978dd2f745aac5eb1fe65c0807eb315" -#KIND_IMAGE ?= "kindest/node:v1.25.3@sha256:f52781bc0d7a19fb6c405c2af83abfeb311f130707a0e219175677e366cc45d1" +KIND_IMAGE ?= "kindest/node:v1.27.3@sha256:3966ac761ae0136263ffdb6cfd4db23ef8a83cba8a463690e98317add2c9ba72" CALICO_TIMEOUT ?= 300s # ---------------------------------------------------------------------------------------------------------------------- @@ -2102,13 +2101,11 @@ uninstall-metallb: ## Uninstall MetalLB install-istio: get-istio ## Install the latest version of Istio into k8s (or override the version using the ISTIO_VERSION env var) $(eval ISTIO_HOME := $(shell find $(TOOLS_DIRECTORY) -maxdepth 1 -type d | grep istio)) $(ISTIO_HOME)/bin/istioctl install --set profile=demo -y - sleep 10 - kubectl -n istio-system get pod -l app=istio-egressgateway -o name | xargs \ - kubectl -n istio-system wait --for condition=ready --timeout 300s - kubectl -n istio-system get pod -l app=istio-ingressgateway -o name | xargs \ - kubectl -n istio-system wait --for condition=ready --timeout 300s - kubectl -n istio-system get pod -l app=istiod -o name | xargs \ - kubectl -n istio-system wait --for condition=ready --timeout 300s + kubectl -n istio-system wait --for condition=available deployment.apps/istiod + kubectl -n istio-system wait --for condition=available deployment.apps/istio-ingressgateway + kubectl -n istio-system wait --for condition=available deployment.apps/istio-egressgateway + kubectl apply -f ./hack/istio-strict.yaml + kubectl -n $(OPERATOR_NAMESPACE) apply -f ./hack/istio-operator.yaml kubectl label namespace $(OPERATOR_NAMESPACE) istio-injection=enabled --overwrite=true # ---------------------------------------------------------------------------------------------------------------------- @@ -2116,6 +2113,8 @@ install-istio: get-istio ## Install the latest version of Istio into k8s (or ove # ---------------------------------------------------------------------------------------------------------------------- .PHONY: uninstall-istio uninstall-istio: get-istio ## Uninstall Istio from k8s + kubectl -n $(OPERATOR_NAMESPACE) delete -f ./hack/istio-operator.yaml || true + kubectl delete -f ./hack/istio-strict.yaml $(eval ISTIO_HOME := $(shell find $(TOOLS_DIRECTORY) -maxdepth 1 -type d | grep istio)) $(ISTIO_HOME)/bin/istioctl uninstall --purge -y diff --git a/api/v1/coherence_types.go b/api/v1/coherence_types.go index 65fdbfd69..fd7d66a0f 100644 --- a/api/v1/coherence_types.go +++ b/api/v1/coherence_types.go @@ -451,6 +451,9 @@ func (in *CoherenceSpec) UpdatePodTemplateSpec(podTemplate *corev1.PodTemplateSp c := EnsureContainerInPod(ContainerNameCoherence, podTemplate) defer ReplaceContainerInPod(podTemplate, c) + lp, localPortAdjust := in.GetLocalPorts() + localPort := fmt.Sprintf("%d", lp) + if in == nil { // we're nil so disable management and metrics/ c.Env = append(c.Env, corev1.EnvVar{Name: EnvVarCohMgmtPrefix + EnvVarCohEnabledSuffix, Value: "false"}, @@ -460,6 +463,9 @@ func (in *CoherenceSpec) UpdatePodTemplateSpec(podTemplate *corev1.PodTemplateSp if deployment.GetType() == CoherenceTypeJob { c.Env = append(c.Env, corev1.EnvVar{Name: EnvVarCohStorage, Value: "false"}) } + + c.Env = append(c.Env, corev1.EnvVar{Name: EnvVarCoherenceLocalPort, Value: localPort}) + c.Env = append(c.Env, corev1.EnvVar{Name: EnvVarCoherenceLocalPortAdjust, Value: localPortAdjust}) return } @@ -472,14 +478,8 @@ func (in *CoherenceSpec) UpdatePodTemplateSpec(podTemplate *corev1.PodTemplateSp } // Always set the unicast ports, as we default them if not specifically set - if in.LocalPort != nil { - c.Env = append(c.Env, corev1.EnvVar{Name: EnvVarCoherenceLocalPort, Value: Int32PtrToString(in.LocalPort)}) - } - - if in.LocalPortAdjust != nil { - lpa := in.LocalPortAdjust - c.Env = append(c.Env, corev1.EnvVar{Name: EnvVarCoherenceLocalPortAdjust, Value: lpa.String()}) - } + c.Env = append(c.Env, corev1.EnvVar{Name: EnvVarCoherenceLocalPort, Value: localPort}) + c.Env = append(c.Env, corev1.EnvVar{Name: EnvVarCoherenceLocalPortAdjust, Value: localPortAdjust}) if in.LogLevel != nil { c.Env = append(c.Env, corev1.EnvVar{Name: EnvVarCohLogLevel, Value: Int32PtrToString(in.LogLevel)}) @@ -525,6 +525,23 @@ func (in *CoherenceSpec) UpdatePodTemplateSpec(podTemplate *corev1.PodTemplateSp in.AddPersistenceVolumes(podTemplate) } +// GetLocalPorts returns the Coherence local ports. +func (in *CoherenceSpec) GetLocalPorts() (int32, string) { + localPort := DefaultUnicastPort + localPortAdjust := fmt.Sprintf("%d", DefaultUnicastPortAdjust) + if in != nil { + if in.LocalPort != nil { + localPort = *in.LocalPort + } + + if in.LocalPortAdjust != nil { + i := *in.LocalPortAdjust + localPortAdjust = i.String() + } + } + return localPort, localPortAdjust +} + // GetMetricsPort returns the metrics port number. func (in *CoherenceSpec) GetMetricsPort() int32 { switch { diff --git a/api/v1/common_test.go b/api/v1/common_test.go index ae4a7470d..cc56a17ca 100644 --- a/api/v1/common_test.go +++ b/api/v1/common_test.go @@ -272,6 +272,14 @@ func createMinimalExpectedPodSpec(deployment coh.CoherenceResource) corev1.PodTe Name: "COH_CLUSTER_NAME", Value: deployment.GetName(), }, + { + Name: "COHERENCE_LOCALPORT", + Value: fmt.Sprintf("%d", coh.DefaultUnicastPort), + }, + { + Name: "COHERENCE_LOCALPORT_ADJUST", + Value: fmt.Sprintf("%d", coh.DefaultUnicastPortAdjust), + }, { Name: "COH_HEALTH_PORT", Value: fmt.Sprintf("%d", spec.GetHealthPort()), diff --git a/docs/installation/01_installation.adoc b/docs/installation/01_installation.adoc index 68b3ca5fa..e3d9fc7c9 100644 --- a/docs/installation/01_installation.adoc +++ b/docs/installation/01_installation.adoc @@ -42,7 +42,20 @@ The prerequisites apply to all installation methods. * A Coherence application image using Coherence version 12.2.1.3 or later. Note that some functionality (e.g. metrics) is only available in Coherence 12.2.1.4 and later. -NOTE: ARM Support: As of version 3.2.0, the Coherence Operator is build as a multi-architecture image that supports running in Kubernetes on both Linux/amd64 and Linux/arm64. The prerequisite is that the Coherence application image used has been built to support ARM. +[NOTE] +==== +ARM Support: As of version 3.2.0, the Coherence Operator is build as a multi-architecture image that supports running in Kubernetes on both Linux/amd64 and Linux/arm64. The prerequisite is that the Coherence application image used has been built to support ARM. +==== + +[NOTE] +==== +Istio (or similar service meshes) + +When installing the Operator and Coherence into Kubernetes cluster that use Istio or similar meshes there are a +number of pre-requisites that must be understood. +See the <> for more details. +==== + There are a number of ways to install the Coherence Operator documented below: diff --git a/examples/400_Istio/README.adoc b/examples/400_Istio/README.adoc index 4cc833083..a46268a23 100644 --- a/examples/400_Istio/README.adoc +++ b/examples/400_Istio/README.adoc @@ -15,22 +15,47 @@ Coherence caches can be accessed from outside the Coherence cluster via Coherenc Using Coherence clusters with Istio does not require the Coherence Operator to also be using Istio (and vice-versa) . The Coherence Operator can manage Coherence clusters independent of whether those clusters are using Istio or not. -[IMPORTANT] -==== -The current support for Istio has the following limitation: +=== Why Doesn't Coherence Work with Istio? -Ports that are exposed in the ports list of the container spec in a Pod will be intercepted by the Envoy proxy in the Istio side-car container. Coherence cluster traffic must not pass through Envoy proxies as this will break Coherence, so the Coherence cluster port must never be exposed as a container port if using Istio. There is no real reason to expose the Coherence cluster port in a container because there is no requirement to have this port externally visible. -==== +Coherence uses a custom TCP message protocol for inter-cluster member communication. +When a cluster member sends a message to another member, the "reply to" address of the sending member is in the message. This address is the socket address the member is listening on (i.e. it is the IP address and port Coherence has bound to). +When Istio is intercepting traffic the message ends up being sent via the Envoy proxy and the actual port Coherence is listening on is blocked by Istio. When the member that receives the message tries to send a response to the reply to address, that port is not visible to it. + +Coherence clients will work with Istio, so Extend, gRPC and http clients for things like REST, metrics and management will work when routed through the Envoy proxy. === Prerequisites The instructions assume that you are using a Kubernetes cluster with Istio installed and configured already. +==== Enable Istio Strict Mode + +For this example we make Istio run in "strict" mode so that it will not allow any traffic between Pods outside the Envoy proxy. If other modes are used, such as permissive, then Coherence will work as normal as its ports will not be blocked. + +To set Istio to strict mode create the following yaml file. + +[source,yaml] +.istio-strict.yaml +---- +apiVersion: security.istio.io/v1beta1 +kind: PeerAuthentication +metadata: + name: "default" +spec: + mtls: + mode: STRICT +---- + +Install this yaml into the Istio system namespace with the following command: + +[source,bash] +---- +kubectl -n istio-system apply istio-strict.yaml +---- + === Using the Coherence operator with Istio To use Coherence operator with Istio, you can deploy the operator into a namespace which has Istio automatic sidecar injection enabled. Before installing the operator, create the namespace in which you want to run the Coherence operator and label it for automatic injection. - [source,bash] ---- kubectl create namespace coherence @@ -39,7 +64,46 @@ kubectl label namespace coherence istio-injection=enabled Istio Sidecar AutoInjection is done automatically when you label the coherence namespace with istio-injection. -After the namespace is labeled, you can install the operator using your preferred method in the Operator <>. +==== Exclude the Operator Web-Hook from the Envoy Proxy + +The Coherence Operator uses an admissions web-hook, which Kubernetes will call to validate Coherence resources. +This web-hook binds to port `9443` in the Operator Pods and is already configured to use TLS as is standard for +Kubernetes admissions web-hooks. If this port is routed through the Envoy proxy Kubernetes will be unable to +access the web-hook. + +There are a number of ways to exclude the web-hook port, the simplest is to add a `PeerAuthentication` resource to the Operator namespace. + +*Before installing the Operator*, create the following `PeerAuthentication` yaml. + +[source,yaml] +.istio-operator.yaml +---- +apiVersion: security.istio.io/v1beta1 +kind: PeerAuthentication +metadata: + name: "coherence-operator" +spec: + selector: + matchLabels: + app.kubernetes.io/name: coherence-operator + app.kubernetes.io/instance: coherence-operator-manager + app.kubernetes.io/component: manager + mtls: + mode: STRICT + portLevelMtls: + 9443: + mode: PERMISSIVE +---- + +Then install this `PeerAuthentication` resource into the same namespace that the Operator will be installed into. +For example, if the Operator will be in the `coherence` namespace: + +[source,bash] +---- +kubectl -n coherence apply istio-operator.yaml +---- + +You can then install the operator using your preferred method in the Operator <>. After installed operator, use the following command to confirm the operator is running: @@ -51,7 +115,7 @@ NAME READY STATUS RESTA coherence-operator-controller-manager-7d76f9f475-q2vwv 2/2 Running 1 17h ---- -2/2 in READY column means that there are 2 containers running in the operator Pod. One is Coherence operator and the other is Envoy Proxy. +The output should show 2/2 in READY column, meaning there are 2 containers running in the Operator pod. One is Coherence Operator and the other is Envoy Proxy. === Creating a Coherence cluster with Istio @@ -63,42 +127,84 @@ kubectl create namespace coherence-example kubectl label namespace coherence-example istio-injection=enabled ---- -There is no other requirements to run Coherence in Istio environment. +==== Exclude the Coherence Cluster Ports -The following is an example that creates a cluster named example-cluster-storage: +As explained above, Coherence cluster traffic must be excluded from the Envoy proxy, there are various ways to do this. -example.yaml -[source,bash] +There are three ports that must be excluded: + +* The cluster port - defaults to 7574, there is no need to set this to any other value. +* The TCP first local port - the Operator will default this to 7575 using its web-hook (if the web-hook is disabled this needs to be manually set). +* The TCP second local port - the Operator will default this to 7576 using its web-hook (if the web-hook is disabled this needs to be manually set). + +*1 Use an Annotation in the Coherence Resource* + +The Istio exclusion annotation `traffic.sidecar.istio.io/excludeInboundPorts` can be added to the Coherence yaml to list the ports to be excluded, + +For example, using the default ports the following annotation will exclude those ports from Istio: + +[source,yaml] +.coherence-storage.yaml ---- -# Example apiVersion: coherence.oracle.com/v1 kind: Coherence metadata: - name: example-cluster-storage + name: storage +spec: + annotations: + traffic.sidecar.istio.io/excludeInboundPorts: "7574,7575,7576" ---- -[source,bash] +If the Coherence Operator's web-hook has been disabled, the local ports must be set in the yaml too: + +[source,yaml] +.coherence-storage.yaml ---- -$ kubectl -n coherence-example apply -f example.yaml +apiVersion: coherence.oracle.com/v1 +kind: Coherence +metadata: + name: storage +spec: + annotations: + traffic.sidecar.istio.io/excludeInboundPorts: "7574,7575,7576" + coherence: + localPort: 7575 + localPortAdjust: 7576 ---- -After you installed the Coherence cluster, run the following command to view the pods: +*2 Use a PeerAuthentication resource* -[source,bash] ----- -$ kubectl -n coherence-example get pods +A `PeerAuthentication` resource can be added to the Coherence cluster's namespace *before the cluster is deployed*. -NAME READY STATUS RESTARTS AGE -example-cluster-storage-0 2/2 Running 0 45m -example-cluster-storage-1 2/2 Running 0 45m -example-cluster-storage-2 2/2 Running 0 45m +[source,yaml] +.istio-coherence.yaml +---- +apiVersion: security.istio.io/v1beta1 +kind: PeerAuthentication +metadata: + name: "coherence" +spec: + selector: + matchLabels: + coherenceComponent: coherencePod + mtls: + mode: STRICT + portLevelMtls: + 7574: + mode: PERMISSIVE + 7575: + mode: PERMISSIVE + 7576: + mode: PERMISSIVE ---- -You can see that 3 members in the cluster are running with 3 pods. 2/2 in READY column means that there are 2 containers running in each Pod. One is Coherence member and the other is Envoy Proxy. +The Coherence Operator labels Coherence Pods with the label `coherenceComponent: coherencePod` so this can be used in the `PeerAuthentication`. Then each port to be excluded is listed in the `portLevelMtls` and set to be `PERMISSIVE`. + +This yaml can then be installed into the namespace that the Coherence cluster will be deployed into. === TLS -Coherence cluster works with mTLS. Coherence client can also support TLS through Istio Gateway with TLS termination to connect to Coherence cluster running inside kubernetes. For example, you can apply the following Istio Gateway and Virtual Service in the namespace of the Coherence cluster. Before applying the gateway, create a secret for the credential from the certificate and key (e.g. server.crt and server.key) to be used by the Gateway: +Coherence clusters work with mTLS and Coherence clients can also support TLS through the Istio Gateway with TLS termination to connect to Coherence cluster running inside kubernetes. For example, you can apply the following Istio Gateway and Virtual Service in the namespace of the Coherence cluster. Before applying the gateway, create a secret for the credential from the certificate and key (e.g. server.crt and server.key) to be used by the Gateway: [source,bash] ---- diff --git a/hack/istio-operator.yaml b/hack/istio-operator.yaml new file mode 100644 index 000000000..04a640db0 --- /dev/null +++ b/hack/istio-operator.yaml @@ -0,0 +1,16 @@ +apiVersion: security.istio.io/v1beta1 +kind: PeerAuthentication +metadata: + name: "coherence-operator" +spec: + selector: + matchLabels: + app.kubernetes.io/name: coherence-operator + app.kubernetes.io/instance: coherence-operator-manager + app.kubernetes.io/component: manager + mtls: + mode: STRICT + portLevelMtls: + 9443: + mode: PERMISSIVE + diff --git a/hack/istio-strict.yaml b/hack/istio-strict.yaml new file mode 100644 index 000000000..4a5134cbc --- /dev/null +++ b/hack/istio-strict.yaml @@ -0,0 +1,9 @@ +apiVersion: security.istio.io/v1beta1 +kind: PeerAuthentication +metadata: + name: "default" + namespace: "istio-system" +spec: + mtls: + mode: STRICT + diff --git a/test/e2e/clients/storage.yaml b/test/e2e/clients/storage.yaml index 2f08055d7..c498b3e3e 100644 --- a/test/e2e/clients/storage.yaml +++ b/test/e2e/clients/storage.yaml @@ -3,6 +3,11 @@ kind: Coherence metadata: name: storage spec: + annotations: + traffic.sidecar.istio.io/excludeInboundPorts: "7574,7575,7576" + coherence: + localPort: 7575 + localPortAdjust: 7576 ports: - name: extend port: 20000