From 18dd58f6930978bb151bea5d171c0e8524423574 Mon Sep 17 00:00:00 2001 From: Andy Taylor Date: Thu, 10 Feb 2022 10:06:59 +0000 Subject: [PATCH] #144 - expose Liveness and Readiness probes --- Makefile | 4 +- api/v1beta1/activemqartemis_types.go | 18 +- api/v1beta1/zz_generated.deepcopy.go | 40 -- .../broker.amq.io_activemqartemises.yaml | 256 ++++++++++- .../activemqartemis_controller_test.go | 405 +++++++++++++++++- controllers/activemqartemis_reconciler.go | 116 ++++- docs/manual/operator.md | 117 ++++- pkg/resources/containers/container.go | 23 - 8 files changed, 868 insertions(+), 111 deletions(-) diff --git a/Makefile b/Makefile index 4969a289c..d7f56f29d 100644 --- a/Makefile +++ b/Makefile @@ -48,6 +48,8 @@ IMG ?= $(OPERATOR_IMAGE_REPO):$(OPERATOR_VERSION) # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. ENVTEST_K8S_VERSION = 1.22 +TEST_ARGS = "" + # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) GOBIN=$(shell go env GOPATH)/bin @@ -94,7 +96,7 @@ vet: ## Run go vet against code. go vet -composites=false ./... test: manifests generate fmt vet envtest ## Run tests. - KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test ./... -coverprofile cover.out + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test ./... $(TEST_ARGS) -coverprofile cover.out ##@ Build diff --git a/api/v1beta1/activemqartemis_types.go b/api/v1beta1/activemqartemis_types.go index 3f64f57c2..5ffdf8541 100644 --- a/api/v1beta1/activemqartemis_types.go +++ b/api/v1beta1/activemqartemis_types.go @@ -208,24 +208,14 @@ type DeploymentPlanType struct { ManagementRBACEnabled bool `json:"managementRBACEnabled,omitempty"` ExtraMounts ExtraMountsType `json:"extraMounts,omitempty"` // Whether broker is clustered - Clustered *bool `json:"clustered,omitempty"` - PodSecurity PodSecurityType `json:"podSecurity,omitempty"` - LivenessProbe LivenessProbeType `json:"livenessProbe,omitempty"` - ReadinessProbe ReadinessProbeType `json:"readinessProbe,omitempty"` + Clustered *bool `json:"clustered,omitempty"` + PodSecurity PodSecurityType `json:"podSecurity,omitempty"` + LivenessProbe corev1.Probe `json:"livenessProbe,omitempty"` + ReadinessProbe corev1.Probe `json:"readinessProbe,omitempty"` // Whether or not to install the artemis metrics plugin EnableMetricsPlugin *bool `json:"enableMetricsPlugin,omitempty"` } -type LivenessProbeType struct { - // Liveness Probe timeoutSeconds for broker container - TimeoutSeconds *int32 `json:"timeoutSeconds,omitempty"` -} - -type ReadinessProbeType struct { - // Readiness Probe timeoutSeconds for broker container - TimeoutSeconds *int32 `json:"timeoutSeconds,omitempty"` -} - type PodSecurityType struct { // ServiceAccount Name of the pod ServiceAccountName *string `json:"serviceAccountName,omitempty"` diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 3a87f795c..9a6e0a777 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -1439,26 +1439,6 @@ func (in *KeycloakModuleConfigurationType) DeepCopy() *KeycloakModuleConfigurati return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *LivenessProbeType) DeepCopyInto(out *LivenessProbeType) { - *out = *in - if in.TimeoutSeconds != nil { - in, out := &in.TimeoutSeconds, &out.TimeoutSeconds - *out = new(int32) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LivenessProbeType. -func (in *LivenessProbeType) DeepCopy() *LivenessProbeType { - if in == nil { - return nil - } - out := new(LivenessProbeType) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LoginModuleReferenceType) DeepCopyInto(out *LoginModuleReferenceType) { *out = *in @@ -1764,26 +1744,6 @@ func (in *QueueConfigurationType) DeepCopy() *QueueConfigurationType { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ReadinessProbeType) DeepCopyInto(out *ReadinessProbeType) { - *out = *in - if in.TimeoutSeconds != nil { - in, out := &in.TimeoutSeconds, &out.TimeoutSeconds - *out = new(int32) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReadinessProbeType. -func (in *ReadinessProbeType) DeepCopy() *ReadinessProbeType { - if in == nil { - return nil - } - out := new(ReadinessProbeType) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RoleAccessType) DeepCopyInto(out *RoleAccessType) { *out = *in diff --git a/config/crd/bases/broker.amq.io_activemqartemises.yaml b/config/crd/bases/broker.amq.io_activemqartemises.yaml index d20502505..43fea7c02 100644 --- a/config/crd/bases/broker.amq.io_activemqartemises.yaml +++ b/config/crd/bases/broker.amq.io_activemqartemises.yaml @@ -537,9 +537,135 @@ spec: description: If aio use ASYNCIO, if nio use NIO for journal IO type: string livenessProbe: + description: Probe describes a health check to be performed against + a container to determine whether it is alive or ready to receive + traffic. properties: + exec: + description: One and only one of the following should be specified. + Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute inside + the container, the working directory for the command is + root ('/') in the container's filesystem. The command + is simply exec'd, it is not run inside a shell, so traditional + shell instructions ('|', etc) won't work. To use a shell, + you need to explicitly call out to that shell. Exit + status of 0 is treated as live/healthy and non-zero + is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe to + be considered failed after having succeeded. Defaults to + 3. Minimum value is 1. + format: int32 + type: integer + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to the + pod IP. You probably want to set "Host" in httpHeaders + instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access on the + container. Number must be in the range 1 to 65535. Name + must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container has started + before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe to + be considered successful after having failed. Defaults to + 1. Must be 1 for liveness and startup. Minimum value is + 1. + format: int32 + type: integer + tcpSocket: + description: 'TCPSocket specifies an action involving a TCP + port. TCP hooks not yet supported TODO: implement a realistic + TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access on the + container. Number must be in the range 1 to 65535. Name + must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs to + terminate gracefully upon probe failure. The grace period + is the duration in seconds after the processes running in + the pod are sent a termination signal and the time when + the processes are forcibly halted with a kill signal. Set + this value longer than the expected cleanup time for your + process. If this value is nil, the pod's terminationGracePeriodSeconds + will be used. Otherwise, this value overrides the value + provided by the pod spec. Value must be non-negative integer. + The value zero indicates stop immediately via the kill signal + (no opportunity to shut down). This is a beta field and + requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is + used if unset. + format: int64 + type: integer timeoutSeconds: - description: Liveness Probe timeoutSeconds for broker container + description: 'Number of seconds after which the probe times + out. Defaults to 1 second. Minimum value is 1. More info: + https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer type: object @@ -565,9 +691,135 @@ spec: type: string type: object readinessProbe: + description: Probe describes a health check to be performed against + a container to determine whether it is alive or ready to receive + traffic. properties: + exec: + description: One and only one of the following should be specified. + Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute inside + the container, the working directory for the command is + root ('/') in the container's filesystem. The command + is simply exec'd, it is not run inside a shell, so traditional + shell instructions ('|', etc) won't work. To use a shell, + you need to explicitly call out to that shell. Exit + status of 0 is treated as live/healthy and non-zero + is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe to + be considered failed after having succeeded. Defaults to + 3. Minimum value is 1. + format: int32 + type: integer + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to the + pod IP. You probably want to set "Host" in httpHeaders + instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header to + be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access on the + container. Number must be in the range 1 to 65535. Name + must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container has started + before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe to + be considered successful after having failed. Defaults to + 1. Must be 1 for liveness and startup. Minimum value is + 1. + format: int32 + type: integer + tcpSocket: + description: 'TCPSocket specifies an action involving a TCP + port. TCP hooks not yet supported TODO: implement a realistic + TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access on the + container. Number must be in the range 1 to 65535. Name + must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs to + terminate gracefully upon probe failure. The grace period + is the duration in seconds after the processes running in + the pod are sent a termination signal and the time when + the processes are forcibly halted with a kill signal. Set + this value longer than the expected cleanup time for your + process. If this value is nil, the pod's terminationGracePeriodSeconds + will be used. Otherwise, this value overrides the value + provided by the pod spec. Value must be non-negative integer. + The value zero indicates stop immediately via the kill signal + (no opportunity to shut down). This is a beta field and + requires enabling ProbeTerminationGracePeriod feature gate. + Minimum value is 1. spec.terminationGracePeriodSeconds is + used if unset. + format: int64 + type: integer timeoutSeconds: - description: Readiness Probe timeoutSeconds for broker container + description: 'Number of seconds after which the probe times + out. Defaults to 1 second. Minimum value is 1. More info: + https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer type: object diff --git a/controllers/activemqartemis_controller_test.go b/controllers/activemqartemis_controller_test.go index 03a05a972..00f0367af 100644 --- a/controllers/activemqartemis_controller_test.go +++ b/controllers/activemqartemis_controller_test.go @@ -20,13 +20,20 @@ package controllers import ( "context" + "math/rand" + "strings" + "fmt" + //"testing" "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -34,41 +41,331 @@ import ( "github.com/artemiscloud/activemq-artemis-operator/pkg/utils/namer" ) +// Uncomment this and the "test" import if you want to debug this set of tests +//func TestArtemisController(t *testing.T) { +// RegisterFailHandler(Fail) +// RunSpecs(t, "Artemis Controller Suite") +// } + var _ = Describe("artemis controller", func() { // Define utility constants for object names and testing timeouts/durations and intervals. const ( - name = "t1" namespace = "default" - timeout = time.Second * 10 + timeout = time.Second * 30 duration = time.Second * 10 interval = time.Millisecond * 250 ) - Context("With delopyed controller", func() { - It("Expect pod desc", func() { - By("By creating a new crd") + Context("Liveness Probe Tests", func() { + It("Override Liveness Probe No Exec", func() { + By("By creating a crd with Liveness Probe") + ctx := context.Background() + crd := generateArtemisSpec(namespace) + crd.Spec.AdminUser = "admin" + crd.Spec.AdminPassword = "password" + livenessProbe := corev1.Probe{} + livenessProbe.PeriodSeconds = 5 + livenessProbe.InitialDelaySeconds = 6 + livenessProbe.TimeoutSeconds = 7 + livenessProbe.SuccessThreshold = 8 + livenessProbe.FailureThreshold = 9 + crd.Spec.DeploymentPlan.LivenessProbe = livenessProbe + createdCrd := &brokerv1beta1.ActiveMQArtemis{} + createdSs := &appsv1.StatefulSet{} + + By("Deploying the CRD") + Expect(k8sClient.Create(ctx, &crd)).Should(Succeed()) + + By("Making sure that the CRD gets deployed") + Eventually(checkCrdCreated(crd.ObjectMeta.Name, namespace, createdCrd), timeout, interval).Should(BeTrue()) + Expect(createdCrd.Name).Should(Equal(crd.ObjectMeta.Name)) + + By("Checking that Stateful Set is Created with the Liveness Probe") + Eventually(func() bool { + key := types.NamespacedName{Name: namer.CrToSS(createdCrd.Name), Namespace: namespace} + + err := k8sClient.Get(ctx, key, createdSs) + + if err != nil { + return false + } + return createdSs.Spec.Template.Spec.Containers[0].LivenessProbe.TCPSocket != nil + }, timeout, interval).Should(Equal(true)) + + By("Making sure the Liveness probe is correct") + Expect(len(createdSs.Spec.Template.Spec.Containers) == 1).Should(BeTrue()) + Expect(createdSs.Spec.Template.Spec.Containers[0].LivenessProbe.Handler.TCPSocket.Port.String() == "8161").Should(BeTrue()) + Expect(createdSs.Spec.Template.Spec.Containers[0].LivenessProbe.PeriodSeconds == 5).Should(BeTrue()) + Expect(createdSs.Spec.Template.Spec.Containers[0].LivenessProbe.InitialDelaySeconds == 6).Should(BeTrue()) + Expect(createdSs.Spec.Template.Spec.Containers[0].LivenessProbe.TimeoutSeconds == 7).Should(BeTrue()) + Expect(createdSs.Spec.Template.Spec.Containers[0].LivenessProbe.SuccessThreshold == 8).Should(BeTrue()) + Expect(createdSs.Spec.Template.Spec.Containers[0].LivenessProbe.FailureThreshold == 9).Should(BeTrue()) + + By("Checking that Stateful Set is updated with the Liveness Probe") + Eventually(func() bool { + key := types.NamespacedName{Name: namer.CrToSS(createdCrd.Name), Namespace: namespace} + + err := k8sClient.Get(ctx, key, createdSs) + + if err != nil { + return false + } + return createdSs.Spec.Template.Spec.Containers[0].LivenessProbe.TCPSocket != nil + }, timeout, interval).Should(Equal(true)) + + By("Updating the CR") + Eventually(checkCrdCreated(crd.ObjectMeta.Name, namespace, createdCrd), timeout, interval).Should(BeTrue()) + original := generateOriginalArtemisSpec(namespace, createdCrd.Name) + + original.Spec.DeploymentPlan.LivenessProbe.PeriodSeconds = 15 + original.Spec.DeploymentPlan.LivenessProbe.InitialDelaySeconds = 16 + original.Spec.DeploymentPlan.LivenessProbe.TimeoutSeconds = 17 + original.Spec.DeploymentPlan.LivenessProbe.SuccessThreshold = 18 + original.Spec.DeploymentPlan.LivenessProbe.FailureThreshold = 19 + exec := corev1.ExecAction{ + Command: []string{"/broker/bin/artemis check node"}, + } + original.Spec.DeploymentPlan.LivenessProbe.Exec = &exec + original.ObjectMeta.ResourceVersion = createdCrd.GetResourceVersion() + By("Redeploying the CRD") + Expect(k8sClient.Update(ctx, original)).Should(Succeed()) + + Eventually(func() bool { + key := types.NamespacedName{Name: namer.CrToSS(createdCrd.Name), Namespace: namespace} + + err := k8sClient.Get(ctx, key, createdSs) + + if err != nil { + return false + } + return createdSs.Spec.Template.Spec.Containers[0].LivenessProbe.PeriodSeconds == 15 + }, timeout, interval).Should(Equal(true)) + + Expect(createdSs.Spec.Template.Spec.Containers[0].LivenessProbe.Handler.Exec != nil).Should(BeTrue()) + Expect(createdSs.Spec.Template.Spec.Containers[0].LivenessProbe.Handler.Exec.Command[0] == "/broker/bin/artemis check node").Should(BeTrue()) + Expect(createdSs.Spec.Template.Spec.Containers[0].LivenessProbe.PeriodSeconds == 15).Should(BeTrue()) + Expect(createdSs.Spec.Template.Spec.Containers[0].LivenessProbe.InitialDelaySeconds == 16).Should(BeTrue()) + Expect(createdSs.Spec.Template.Spec.Containers[0].LivenessProbe.TimeoutSeconds == 17).Should(BeTrue()) + Expect(createdSs.Spec.Template.Spec.Containers[0].LivenessProbe.SuccessThreshold == 18).Should(BeTrue()) + Expect(createdSs.Spec.Template.Spec.Containers[0].LivenessProbe.FailureThreshold == 19).Should(BeTrue()) + + By("check it has gone") + Expect(k8sClient.Delete(ctx, createdCrd)) + Eventually(checkCrdDeleted(crd.ObjectMeta.Name, namespace, createdCrd), timeout, interval).Should(BeTrue()) + }) + + It("Override Liveness Probe Exec", func() { + By("By creating a crd with Liveness Probe") ctx := context.Background() - crd := &brokerv1beta1.ActiveMQArtemis{ - TypeMeta: metav1.TypeMeta{ - Kind: "ActiveMQArtemis", - APIVersion: brokerv1beta1.GroupVersion.Identifier(), - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "t1", - Namespace: namespace, - }, + crd := generateArtemisSpec(namespace) + crd.Spec.AdminUser = "admin" + crd.Spec.AdminPassword = "password" + exec := corev1.ExecAction{ + Command: []string{"/broker/bin/artemis check node"}, } - Expect(k8sClient.Create(ctx, crd)).Should(Succeed()) + livenessProbe := corev1.Probe{} + livenessProbe.Exec = &exec + crd.Spec.DeploymentPlan.LivenessProbe = livenessProbe + createdCrd := &brokerv1beta1.ActiveMQArtemis{} + createdSs := &appsv1.StatefulSet{} + By("Deploying the CRD") + Expect(k8sClient.Create(ctx, &crd)).Should(Succeed()) + By("Making sure that the CRD gets deployed") + Eventually(checkCrdCreated(crd.ObjectMeta.Name, namespace, createdCrd), timeout, interval).Should(BeTrue()) + Expect(createdCrd.Name).Should(Equal(crd.ObjectMeta.Name)) + + By("Checking that Stateful Set is Created with the Liveness Probe") + Eventually(func() bool { + key := types.NamespacedName{Name: namer.CrToSS(createdCrd.Name), Namespace: namespace} + + err := k8sClient.Get(ctx, key, createdSs) + + if err != nil { + return false + } + return createdSs.Spec.Template.Spec.Containers[0].LivenessProbe.Exec != nil + }, timeout, interval).Should(Equal(true)) + + By("Making sure the Liveness probe is correct") + Expect(len(createdSs.Spec.Template.Spec.Containers) == 1).Should(BeTrue()) + Expect(createdSs.Spec.Template.Spec.Containers[0].LivenessProbe.Handler.Exec.Command[0] == "/broker/bin/artemis check node").Should(BeTrue()) + Expect(k8sClient.Delete(ctx, createdCrd)) + + By("check it has gone") + Eventually(checkCrdDeleted(crd.ObjectMeta.Name, namespace, createdCrd), timeout, interval).Should(BeTrue()) + }) + + It("Default Liveness Probe", func() { + By("By creating a crd without Liveness Probe") + ctx := context.Background() + crd := generateArtemisSpec(namespace) + crd.Spec.AdminUser = "admin" + crd.Spec.AdminPassword = "password" createdCrd := &brokerv1beta1.ActiveMQArtemis{} + createdSs := &appsv1.StatefulSet{} + Expect(k8sClient.Create(ctx, &crd)).Should(Succeed()) + Eventually(checkCrdCreated(crd.ObjectMeta.Name, namespace, createdCrd), timeout, interval).Should(BeTrue()) + Expect(createdCrd.Name).Should(Equal(crd.ObjectMeta.Name)) + By("Checking that the Liveness Probe is created") Eventually(func() bool { - key := types.NamespacedName{Name: name, Namespace: namespace} - err := k8sClient.Get(ctx, key, createdCrd) - return err == nil - }, timeout, interval).Should(BeTrue()) - Expect(createdCrd.Name).Should(Equal(name)) + key := types.NamespacedName{Name: namer.CrToSS(createdCrd.Name), Namespace: namespace} + + err := k8sClient.Get(ctx, key, createdSs) + + if err != nil { + return false + } + return createdSs.Spec.Template.Spec.Containers[0].LivenessProbe.TCPSocket != nil + }, timeout, interval).Should(Equal(true)) + + Expect(len(createdSs.Spec.Template.Spec.Containers) == 1).Should(BeTrue()) + Expect(createdSs.Spec.Template.Spec.Containers[0].LivenessProbe.Handler.TCPSocket.Port.String() == "8161").Should(BeTrue()) + Expect(k8sClient.Delete(ctx, createdCrd)) + + By("check it has gone") + Eventually(checkCrdDeleted(crd.ObjectMeta.Name, namespace, createdCrd), timeout, interval).Should(BeTrue()) + }) + }) + + Context("Readiness Probe Tests", func() { + It("Override Readiness Probe No Exec", func() { + By("By creating a crd with Readiness Probe") + ctx := context.Background() + crd := generateArtemisSpec(namespace) + crd.Spec.AdminUser = "admin" + crd.Spec.AdminPassword = "password" + readinessProbe := corev1.Probe{} + readinessProbe.PeriodSeconds = 5 + readinessProbe.InitialDelaySeconds = 6 + readinessProbe.TimeoutSeconds = 7 + readinessProbe.SuccessThreshold = 8 + readinessProbe.FailureThreshold = 9 + crd.Spec.DeploymentPlan.ReadinessProbe = readinessProbe + createdCrd := &brokerv1beta1.ActiveMQArtemis{} + createdSs := &appsv1.StatefulSet{} + By("Deploying the CRD") + Expect(k8sClient.Create(ctx, &crd)).Should(Succeed()) + + By("Making sure that the CRD gets deployed") + Eventually(checkCrdCreated(crd.ObjectMeta.Name, namespace, createdCrd), timeout, interval).Should(BeTrue()) + Expect(createdCrd.Name).Should(Equal(crd.ObjectMeta.Name)) + + By("Checking that Stateful Set is Created with the Readiness Probe") + Eventually(func() bool { + key := types.NamespacedName{Name: namer.CrToSS(createdCrd.Name), Namespace: namespace} + + err := k8sClient.Get(ctx, key, createdSs) + + if err != nil { + return false + } + return createdSs.Spec.Template.Spec.Containers[0].ReadinessProbe.Exec != nil + }, timeout, interval).Should(Equal(true)) + + By("Making sure the Readiness probe is correct") + Expect(len(createdSs.Spec.Template.Spec.Containers) == 1).Should(BeTrue()) + Expect(createdSs.Spec.Template.Spec.Containers[0].ReadinessProbe.Handler.Exec.Command[0] == "/bin/bash").Should(BeTrue()) + Expect(createdSs.Spec.Template.Spec.Containers[0].ReadinessProbe.Handler.Exec.Command[1] == "-c").Should(BeTrue()) + Expect(createdSs.Spec.Template.Spec.Containers[0].ReadinessProbe.Handler.Exec.Command[2] == "/opt/amq/bin/readinessProbe.sh").Should(BeTrue()) + Expect(k8sClient.Delete(ctx, createdCrd)) + + By("check it has gone") + Eventually(checkCrdDeleted(crd.ObjectMeta.Name, namespace, createdCrd), timeout, interval).Should(BeTrue()) + }) + + It("Override Readiness Probe Exec", func() { + By("By creating a crd with Readiness Probe") + ctx := context.Background() + crd := generateArtemisSpec(namespace) + crd.Spec.AdminUser = "admin" + crd.Spec.AdminPassword = "password" + exec := corev1.ExecAction{ + Command: []string{"/broker/bin/artemis check node"}, + } + readinessProbe := corev1.Probe{} + readinessProbe.Exec = &exec + crd.Spec.DeploymentPlan.ReadinessProbe = readinessProbe + createdCrd := &brokerv1beta1.ActiveMQArtemis{} + createdSs := &appsv1.StatefulSet{} + By("Deploying the CRD") + Expect(k8sClient.Create(ctx, &crd)).Should(Succeed()) + + By("Making sure that the CRD gets deployed") + Eventually(checkCrdCreated(crd.ObjectMeta.Name, namespace, createdCrd), timeout, interval).Should(BeTrue()) + Expect(createdCrd.Name).Should(Equal(crd.ObjectMeta.Name)) + + By("Checking that Stateful Set is Created with the Readiness Probe") + Eventually(func() bool { + key := types.NamespacedName{Name: namer.CrToSS(createdCrd.Name), Namespace: namespace} + + err := k8sClient.Get(ctx, key, createdSs) + + if err != nil { + return false + } + return createdSs.Spec.Template.Spec.Containers[0].ReadinessProbe.Exec != nil + }, timeout, interval).Should(Equal(true)) + + By("Making sure the Readiness probe is correct") + Expect(len(createdSs.Spec.Template.Spec.Containers) == 1).Should(BeTrue()) + Expect(createdSs.Spec.Template.Spec.Containers[0].ReadinessProbe.Handler.Exec.Command[0] == "/broker/bin/artemis check node").Should(BeTrue()) + Expect(k8sClient.Delete(ctx, createdCrd)) + + By("check it has gone") + Eventually(checkCrdDeleted(crd.ObjectMeta.Name, namespace, createdCrd), timeout, interval).Should(BeTrue()) + }) + + It("Default Readiness Probe", func() { + By("By creating a crd without Readiness Probe") + ctx := context.Background() + crd := generateArtemisSpec(namespace) + crd.Spec.AdminUser = "admin" + crd.Spec.AdminPassword = "password" + createdCrd := &brokerv1beta1.ActiveMQArtemis{} + createdSs := &appsv1.StatefulSet{} + Expect(k8sClient.Create(ctx, &crd)).Should(Succeed()) + + Eventually(checkCrdCreated(crd.ObjectMeta.Name, namespace, createdCrd), timeout, interval).Should(BeTrue()) + Expect(createdCrd.Name).Should(Equal(crd.ObjectMeta.Name)) + + By("Checking that the Readiness Probe is created") + Eventually(func() bool { + key := types.NamespacedName{Name: namer.CrToSS(createdCrd.Name), Namespace: namespace} + + err := k8sClient.Get(ctx, key, createdSs) + + if err != nil { + return false + } + return createdSs.Spec.Template.Spec.Containers[0].ReadinessProbe.Exec != nil + }, timeout, interval).Should(Equal(true)) + + By("Making sure the Readiness probe is correct") + Expect(len(createdSs.Spec.Template.Spec.Containers) == 1).Should(BeTrue()) + Expect(createdSs.Spec.Template.Spec.Containers[0].ReadinessProbe.Handler.Exec.Command[0] == "/bin/bash").Should(BeTrue()) + Expect(createdSs.Spec.Template.Spec.Containers[0].ReadinessProbe.Handler.Exec.Command[1] == "-c").Should(BeTrue()) + Expect(createdSs.Spec.Template.Spec.Containers[0].ReadinessProbe.Handler.Exec.Command[2] == "/opt/amq/bin/readinessProbe.sh").Should(BeTrue()) + Expect(k8sClient.Delete(ctx, createdCrd)) + + By("check it has gone") + Eventually(checkCrdDeleted(crd.ObjectMeta.Name, namespace, createdCrd), timeout, interval).Should(BeTrue()) + }) + }) + + Context("With deployed controller", func() { + It("Expect pod desc", func() { + By("By creating a new crd") + ctx := context.Background() + crd := generateArtemisSpec(namespace) + Expect(k8sClient.Create(ctx, &crd)).Should(Succeed()) + + createdCrd := &brokerv1beta1.ActiveMQArtemis{} + + Eventually(checkCrdCreated(crd.ObjectMeta.Name, namespace, createdCrd), timeout, interval).Should(BeTrue()) + Expect(createdCrd.Name).Should(Equal(crd.ObjectMeta.Name)) // would like more status updates on createdCrd @@ -90,7 +387,7 @@ var _ = Describe("artemis controller", func() { By("Checking stopped status of CR because we expect it to fail to deploy") Eventually(func() (int, error) { - key := types.NamespacedName{Name: name, Namespace: namespace} + key := types.NamespacedName{Name: crd.ObjectMeta.Name, Namespace: namespace} err := k8sClient.Get(ctx, key, createdCrd) if err != nil { @@ -104,7 +401,73 @@ var _ = Describe("artemis controller", func() { } return len(createdCrd.Status.PodStatus.Stopped), nil }, timeout, interval).Should(Equal(1)) + Expect(k8sClient.Delete(ctx, &crd)).Should(Succeed()) + By("check it has gone") + Eventually(checkCrdDeleted(crd.ObjectMeta.Name, namespace, createdCrd), timeout, interval).Should(BeTrue()) }) }) + }) + +func generateArtemisSpec(namespace string) brokerv1beta1.ActiveMQArtemis { + + spec := brokerv1beta1.ActiveMQArtemisSpec{} + + toCreate := brokerv1beta1.ActiveMQArtemis{ + TypeMeta: metav1.TypeMeta{ + Kind: "ActiveMQArtemis", + APIVersion: brokerv1beta1.GroupVersion.Identifier(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: randString(), + Namespace: namespace, + }, + Spec: spec, + } + + return toCreate +} + +func generateOriginalArtemisSpec(namespace string, name string) *brokerv1beta1.ActiveMQArtemis { + + spec := brokerv1beta1.ActiveMQArtemisSpec{} + + toCreate := brokerv1beta1.ActiveMQArtemis{ + TypeMeta: metav1.TypeMeta{ + Kind: "ActiveMQArtemis", + APIVersion: brokerv1beta1.GroupVersion.Identifier(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: spec, + } + + return &toCreate +} + +func randString() string { + rand.Seed(time.Now().UnixNano()) + chars := []rune("abcdefghijklmnopqrstuvwxyz") + length := 6 + var b strings.Builder + b.WriteString("broker-") + for i := 0; i < length; i++ { + b.WriteRune(chars[rand.Intn(len(chars))]) + } + return b.String() +} + +func checkCrdCreated(name string, nameSpace string, crd *brokerv1beta1.ActiveMQArtemis) bool { + key := types.NamespacedName{Name: name, Namespace: nameSpace} + err := k8sClient.Get(ctx, key, crd) + return err == nil +} + +func checkCrdDeleted(name string, namespace string, crd *brokerv1beta1.ActiveMQArtemis) bool { + //fetched := &pscv1alpha1.PreScaledCronJob{} + err := k8sClient.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, crd) + return errors.IsNotFound(err) +} diff --git a/controllers/activemqartemis_reconciler.go b/controllers/activemqartemis_reconciler.go index b7e4fb73e..4a45ecded 100644 --- a/controllers/activemqartemis_reconciler.go +++ b/controllers/activemqartemis_reconciler.go @@ -28,6 +28,7 @@ import ( "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" ctrl "sigs.k8s.io/controller-runtime" rtclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -66,6 +67,9 @@ const ( statefulSetConnectorsUpdated = 1 << 9 statefulSetConsoleUpdated = 1 << 10 statefulSetInitImageUpdated = 1 << 11 + + livenessProbeGraceTime = 30 + TCPLivenessPort = 8161 ) var defaultMessageMigration bool = true @@ -217,6 +221,12 @@ func (reconciler *ActiveMQArtemisReconcilerImpl) ProcessStatefulSet(fsm *ActiveM fsm.SetPodInvalid(false) newPodTemplateCreated = true } + + if !processLivenessProbe(fsm.customResource, fsm.prevCustomResource) || !processReadinessProbe(fsm.customResource, fsm.prevCustomResource) { + *fsm.prevCustomResource = *fsm.customResource + currentStatefulSet.Spec.Template = NewPodTemplateSpecForCR(fsm) + newPodTemplateCreated = true + } } labels := fsm.namers.LabelBuilder.Labels() @@ -388,6 +398,14 @@ func (reconciler *ActiveMQArtemisReconcilerImpl) ProcessAddressSettings(customRe return compareAddressSettings(&prevCustomResource.Spec.AddressSettings, &customResource.Spec.AddressSettings) } +func processLivenessProbe(customResource *brokerv1beta1.ActiveMQArtemis, prevCustomResource *brokerv1beta1.ActiveMQArtemis) bool { + return reflect.DeepEqual(prevCustomResource.Spec.DeploymentPlan.LivenessProbe, customResource.Spec.DeploymentPlan.LivenessProbe) +} + +func processReadinessProbe(customResource *brokerv1beta1.ActiveMQArtemis, prevCustomResource *brokerv1beta1.ActiveMQArtemis) bool { + return reflect.DeepEqual(prevCustomResource.Spec.DeploymentPlan.ReadinessProbe, customResource.Spec.DeploymentPlan.ReadinessProbe) +} + //returns true if currentAddressSettings need update func compareAddressSettings(currentAddressSettings *brokerv1beta1.AddressSettingsType, newAddressSettings *brokerv1beta1.AddressSettingsType) bool { @@ -1791,15 +1809,8 @@ func NewPodTemplateSpecForCR(fsm *ActiveMQArtemisFSM) corev1.PodTemplateSpec { } reqLogger.V(1).Info("now mounts added to container", "new len", len(container.VolumeMounts)) - if fsm.customResource.Spec.DeploymentPlan.LivenessProbe.TimeoutSeconds != nil { - container.LivenessProbe.TimeoutSeconds = *fsm.customResource.Spec.DeploymentPlan.LivenessProbe.TimeoutSeconds - reqLogger.V(1).Info("Setting liveness timeoutSeconds", "value", container.LivenessProbe.TimeoutSeconds) - } - reqLogger.V(1).Info("Checking out readiness", "crvalue", fsm.customResource.Spec.DeploymentPlan.ReadinessProbe.TimeoutSeconds) - if fsm.customResource.Spec.DeploymentPlan.ReadinessProbe.TimeoutSeconds != nil { - container.ReadinessProbe.TimeoutSeconds = *fsm.customResource.Spec.DeploymentPlan.ReadinessProbe.TimeoutSeconds - reqLogger.V(1).Info("Setting readiness timeoutSeconds", "value", container.ReadinessProbe.TimeoutSeconds) - } + container.LivenessProbe = configureLivenessProbe(&fsm.customResource.Spec.DeploymentPlan.LivenessProbe) + container.ReadinessProbe = configureReadinessProbe(&fsm.customResource.Spec.DeploymentPlan.ReadinessProbe) Spec.Containers = append(Containers, container) brokerVolumes := MakeVolumes(fsm) @@ -2018,6 +2029,93 @@ func NewPodTemplateSpecForCR(fsm *ActiveMQArtemisFSM) corev1.PodTemplateSpec { return pts } +func configureLivenessProbe(probe *corev1.Probe) *corev1.Probe { + clog.V(1).Info("Creating Liveness Probe") + if probe != nil { + //copy the probe + + clog.V(1).Info("Liveness Probe provided by user") + livenessProbe := &corev1.Probe{ + InitialDelaySeconds: probe.InitialDelaySeconds, + TimeoutSeconds: probe.TimeoutSeconds, + PeriodSeconds: probe.PeriodSeconds, + TerminationGracePeriodSeconds: probe.TerminationGracePeriodSeconds, + SuccessThreshold: probe.SuccessThreshold, + FailureThreshold: probe.FailureThreshold, + } + if probe.Exec == nil && probe.HTTPGet == nil && probe.TCPSocket == nil { + clog.V(1).Info("Adding default TCP check") + livenessProbe.Handler = corev1.Handler{ + TCPSocket: &corev1.TCPSocketAction{ + Port: intstr.FromInt(TCPLivenessPort), + }, + } + } else { + clog.V(1).Info("Using user provided Liveness Probe " + probe.Exec.String()) + livenessProbe.Exec = probe.Exec + } + return livenessProbe + } else { + livenessProbe := &corev1.Probe{ + InitialDelaySeconds: livenessProbeGraceTime, + TimeoutSeconds: 5, + Handler: corev1.Handler{ + TCPSocket: &corev1.TCPSocketAction{ + Port: intstr.FromInt(TCPLivenessPort), + }, + }, + } + clog.V(1).Info("Creating Default Liveness Probe") + return livenessProbe + } +} + +func configureReadinessProbe(probe *corev1.Probe) *corev1.Probe { + if probe != nil { + //copy the probe + readinessProbe := &corev1.Probe{ + InitialDelaySeconds: probe.InitialDelaySeconds, + TimeoutSeconds: probe.TimeoutSeconds, + PeriodSeconds: probe.PeriodSeconds, + TerminationGracePeriodSeconds: probe.TerminationGracePeriodSeconds, + SuccessThreshold: probe.SuccessThreshold, + FailureThreshold: probe.FailureThreshold, + } + if probe.Exec == nil && probe.HTTPGet == nil && probe.TCPSocket == nil { + //add the default readiness check if none + clog.V(1).Info("Using user provided readiness Probe") + readinessProbe.Handler = corev1.Handler{ + Exec: &corev1.ExecAction{ + Command: []string{ + "/bin/bash", + "-c", + "/opt/amq/bin/readinessProbe.sh", + }, + }, + } + } else { + readinessProbe.Handler = probe.Handler + } + return readinessProbe + } else { + readinessProbe := &corev1.Probe{ + InitialDelaySeconds: livenessProbeGraceTime, + TimeoutSeconds: 5, + Handler: corev1.Handler{ + Exec: &corev1.ExecAction{ + Command: []string{ + "/bin/bash", + "-c", + "/opt/amq/bin/readinessProbe.sh", + }, + }, + }, + } + clog.V(1).Info("Creating Default readiness Probe") + return readinessProbe + } +} + func configPodSecurity(podSpec *corev1.PodSpec, podSecurity *brokerv1beta1.PodSecurityType) { if podSecurity.ServiceAccountName != nil { clog.Info("Pod serviceAccountName specified", "existing", podSpec.ServiceAccountName, "new", *podSecurity.ServiceAccountName) diff --git a/docs/manual/operator.md b/docs/manual/operator.md index e8dfad601..7042a5c05 100644 --- a/docs/manual/operator.md +++ b/docs/manual/operator.md @@ -230,4 +230,119 @@ the broker and addressing are running, and that these controllers have started s It is recommended that you deploy only a single instance of the ActiveMQ Artemis Operator in a given Kubernetes project. Setting the replicas element of your Operator deployment to a value greater than 1, or deploying the Operator more than -once in the same project is not recommended. \ No newline at end of file +once in the same project is not recommended. + +## Configuring the Liveness and Readiness Probe + +The Liveness and readiness Probes are used by Kubernetes to detect when the Broker is started and to check it is still alive. +For full documentation on this topic refer to the [Configure Liveness, Readiness and Startup Probes](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/) +chapter in the Kubernetes documentation. + +### The Liveness probe + +The Liveness probe is configured in the Artemis CR something like: + +```yaml +spec: + deploymentPlan: + size: 1 + image: placeholder + livenessProbe: + initialDelaySeconds: 5 + periodSeconds: 5 +``` + +If no Liveness probe is configured or the handler itself is missing from a configured Liveness Probe then the Operator +will create a default TCP Probe that will check the liveness of the broker by connecting to the web Server port, the default config is: + +```yaml +spec: + deploymentPlan: + livenessProbe: + tcpSocket: + port: 8181 + initialDelaySeconds: 30, + timeoutSeconds: 5, +``` + +#### Using the Artemis Health Check + +you can also use the Artemis Health Checker to check that the broker is running, something like: + +```yaml +spec: + deploymentPlan: + livenessProbe: + exec: + command: + - /home/jboss/amq-broker/bin/artemis + - check + - node + - --silent + - --user + - $AMQ_USER + - --password + - $AMQ_PASSWORD + initialDelaySeconds: 30, + timeoutSeconds: + +``` + +By default this uses the URI of the acceptor configured with the name **artemis**. Since this is not configured by default +it will need configuring in the broker CR. Alternatively configure the acceptor used by passing the **--acceptor** +argument on the artemis check command. + + + NOTE: $AMQ_USER and $AMQ_PASSWORD are environment variables that are configured by the Operator + +You can also check the status of the broker by producing and consuming a message: + +```yaml +spec: + deploymentPlan: + livenessProbe: + exec: + command: + - /home/jboss/amq-broker/bin/artemis + - check + - queue + - --name + - livenessqueue + - --produce + - "1" + - --consume + - "1" + - --silent + - --user + - $AMQ_USER + - --password + - $AMQ_PASSWORD + initialDelaySeconds: 30, + timeoutSeconds: +``` + +The liveness queue must exist and be deployed the broker and be of type anycast with acceptable configuration, something like: + +```yaml +apiVersion: broker.amq.io/v1beta1 +kind: ActiveMQArtemisAddress +metadata: + name: livenessqueue + namespace: activemq-artemis-operator +spec: + addressName: livenessqueue + queueConfiguration: + purgeOnNoConsumers: false + maxConsumers: -1 + durable: true + enabled: true + queueName: livenessqueue + routingType: anycast +``` + +### The Readiness Probe + +As with the Liveness Probe the Readiness probe has a default probe if not configured. Unlike the readiness probe this is +a script that is shipped in the Kubernetes Image, this can be found [here](https://github.com/artemiscloud/activemq-artemis-broker-kubernetes-image/blob/main/modules/activemq-artemis-launch/added/readinessProbe.sh) + +The script will try to establish a tcp connection to each port configured in the broker.xml. \ No newline at end of file diff --git a/pkg/resources/containers/container.go b/pkg/resources/containers/container.go index a9b737711..ab232725d 100644 --- a/pkg/resources/containers/container.go +++ b/pkg/resources/containers/container.go @@ -2,7 +2,6 @@ package containers import ( corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/intstr" ) const ( @@ -18,28 +17,6 @@ func MakeContainer(customResourceName string, imageName string, envVarArray []co Image: imageName, //cr.Spec.DeploymentPlan.Image, Command: []string{"/opt/amq/bin/launch.sh", "start"}, Env: envVarArray, //environments.MakeEnvVarArrayForCR(cr), - ReadinessProbe: &corev1.Probe{ - InitialDelaySeconds: graceTime, - TimeoutSeconds: 5, - Handler: corev1.Handler{ - Exec: &corev1.ExecAction{ - Command: []string{ - "/bin/bash", - "-c", - "/opt/amq/bin/readinessProbe.sh", - }, - }, - }, - }, - LivenessProbe: &corev1.Probe{ - InitialDelaySeconds: graceTime, - TimeoutSeconds: 5, - Handler: corev1.Handler{ - TCPSocket: &corev1.TCPSocketAction{ - Port: intstr.FromInt(TCPLivenessPort), - }, - }, - }, } return container