From 24b12e02b970b27c5701be4dd45053dd9d03a436 Mon Sep 17 00:00:00 2001 From: Tobias Persson Date: Thu, 10 Oct 2024 14:04:44 +0200 Subject: [PATCH 01/11] Add the suite starter to an ETOS cluster This will enable us to run both versions of ETOS using the same ETOS cluster. --- api/v1alpha1/cluster_types.go | 84 ++- api/v1alpha1/common_types.go | 6 +- api/v1alpha1/testrun_webhook.go | 2 +- api/v1alpha1/zz_generated.deepcopy.go | 49 ++ ...s.eiffel-community.github.io_clusters.yaml | 105 ++- ...mmunity.github.io_environmentrequests.yaml | 2 +- ...ffel-community.github.io_environments.yaml | 2 +- ....eiffel-community.github.io_providers.yaml | 2 +- ...s.eiffel-community.github.io_testruns.yaml | 6 +- config/manager/kustomization.yaml | 4 + .../environmentrequest_controller.go | 2 +- internal/etos/api/api.go | 75 ++- internal/etos/etos.go | 10 +- internal/etos/suitestarter/suitestarter.go | 635 ++++++++++++++++++ internal/extras/eventrepository.go | 4 + internal/extras/messagebus.go | 6 - internal/extras/rabbitmq.go | 6 - 17 files changed, 957 insertions(+), 43 deletions(-) create mode 100644 internal/etos/suitestarter/suitestarter.go diff --git a/api/v1alpha1/cluster_types.go b/api/v1alpha1/cluster_types.go index 61301c81..6c1e81d9 100644 --- a/api/v1alpha1/cluster_types.go +++ b/api/v1alpha1/cluster_types.go @@ -20,6 +20,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +// MongoDB describes the deployment of a MongoDB. type MongoDB struct { // +kubebuilder:default=false // +optional @@ -30,6 +31,7 @@ type MongoDB struct { URI Var `json:"uri"` } +// EventRepository describes the deployment of an event repository for ETOS. type EventRepository struct { // Deploy a local event repository for a cluster. // +kubebuilder:default=false @@ -47,6 +49,12 @@ type EventRepository struct { // +kubebuilder:default={"image": "registry.nordix.org/eiffel/eiffel-graphql-storage:latest"} // +optional Storage Image `json:"storage"` + // +kubebuilder:default="etos" + // +optional + EiffelQueueName string `json:"eiffelQueueName,omitempty"` + // +kubebuilder:default="" + // +optional + EiffelQueueParams string `json:"eiffelQueueParams,omitempty"` // +kubebuilder:default={} // +optional @@ -59,15 +67,15 @@ type EventRepository struct { Ingress Ingress `json:"ingress"` } +// MessageBus describes the deployment of messagesbuses for ETOS. type MessageBus struct { - // +kubebuilder:default={"queueName": "etos"} // +optional EiffelMessageBus RabbitMQ `json:"eiffel"` - // +kubebuilder:default={"queueName": "etos-*-temp"} // +optional ETOSMessageBus RabbitMQ `json:"logs"` } +// Etcd describes the deployment of an ETCD database. type Etcd struct { // Parameter is ignored if Deploy is set to true. // +kubebuilder:default="etcd-client" @@ -79,6 +87,7 @@ type Etcd struct { Port string `json:"port"` } +// Database describes the deployment of a database for ETOS. type Database struct { // +kubebuilder:default=true // +optional @@ -88,31 +97,96 @@ type Database struct { Etcd Etcd `json:"etcd"` } +// ETOSAPI describes the deployment of the ETOS API. type ETOSAPI struct { Image `json:",inline"` + // The provider secrets are necessary in order to run ETOS the old way and not using the controller. + // They can be removed from here when the suite starter is no longer in use. + // +optional + IUTProviderSecret string `json:"iutProviderSecret"` + // +optional + ExecutionSpaceProviderSecret string `json:"executionSpaceProviderSecret"` + // +optional + LogAreaProviderSecret string `json:"logAreaProviderSecret"` +} + +// ETOSSuiteStarterConfig describes the configuration required for a suite starter. +// This is separate from the ETOSConfig as we want to remove this in the future when the suite +// starter is no longer in use. +type ETOSSuiteStarterConfig struct { + // +kubebuilder:default="3600" + // +optional + TTL string `json:"ttl,omitempty"` + // +kubebuilder:default="" + // +optional + ObservabilityConfigmapName string `json:"observabilityConfigmapName,omitempty"` + // +kubebuilder:default="300" + // +optional + GracePeriod string `json:"gracePeriod,omitempty"` + // +kubebuilder:default="" + // +optional + SidecarImage string `json:"sidecarImage,omitempty"` + // +kubebuilder:default="" + // +optional + OTELCollectorHost string `json:"otelCollectorHost,omitempty"` } +// ETOSSuiteStarter describes the deployment of an ETOS suite starter. +type ETOSSuiteStarter struct { + Image `json:",inline"` + // +kubebuilder:default="etos-suite-starter" + // +optional + EiffelQueueName string `json:"eiffelQueueName,omitempty"` + // +kubebuilder:default="" + // +optional + EiffelQueueParams string `json:"eiffelQueueParams,omitempty"` + // Provide a custom suite runner template. + // +kubebuilder:default="" + // +optional + SuiteRunnerTemplateConfigmapName string `json:"suiteRunnerTemplateConfigmapName,omitempty"` + // +kubebuilder:default={"ttl": "3600", "gracePeriod": "300"} + // +optional + Config ETOSSuiteStarterConfig `json:"config"` +} + +// ETOSSSE describes th deployment of an ETOS SSE API. type ETOSSSE struct { Image `json:",inline"` } +// ETOSLogArea describes th deployment of an ETOS log area API. type ETOSLogArea struct { Image `json:",inline"` } +// ETOSLogListener describes the deployment of an ETOS log listener. +type ETOSLogListener struct { + Image `json:",inline"` + // +kubebuilder:default="etos-*-temp" + // +optional + ETOSQueueName string `json:"etosQueueName,omitempty"` + // +kubebuilder:default="" + // +optional + ETOSQueueParams string `json:"etosQueueParams,omitempty"` +} + +// ETOSSuiteRunner describes the deployment of an ETOS suite runner. type ETOSSuiteRunner struct { Image `json:",inline"` - LogListener Image `json:"logListener"` + LogListener ETOSLogListener `json:"logListener"` } +// ETOSTestRunner describes the deployment of an ETOS test runner. type ETOSTestRunner struct { Version string `json:"version"` } +// ETOSEnvironmentProvider describes the deployment of an ETOS environment provider. type ETOSEnvironmentProvider struct { Image `json:",inline"` } +// ETOSConfig describes a common configuration for ETOS. type ETOSConfig struct { // +kubebuilder:default="true" // +optional @@ -146,6 +220,7 @@ type ETOSConfig struct { Timezone string `json:"timezone,omitempty"` } +// ETOS describes the deployment of an ETOS cluster. type ETOS struct { // +kubebuilder:default={"image": "registry.nordix.org/eiffel/etos-api:latest"} // +optional @@ -162,6 +237,9 @@ type ETOS struct { // +kubebuilder:default={"version": "latest"} // +optional TestRunner ETOSTestRunner `json:"testRunner"` + // +kubebuilder:default={"image": "registry.nordix.org/eiffel/etos-suite-starter:latest"} + // +optional + SuiteStarter ETOSSuiteStarter `json:"suiteStarter"` // +kubebuilder:default={"image": "registry.nordix.org/eiffel/etos-environment-provider:latest"} // +optional EnvironmentProvider ETOSEnvironmentProvider `json:"environmentProvider"` diff --git a/api/v1alpha1/common_types.go b/api/v1alpha1/common_types.go index 9e520606..dccd9693 100644 --- a/api/v1alpha1/common_types.go +++ b/api/v1alpha1/common_types.go @@ -127,7 +127,7 @@ type RabbitMQ struct { SSL string `json:"ssl"` // +kubebuilder:default=/ // +optional - Vhost string `json:"vhost"` - QueueName string `json:"queueName,omitempty"` - QueueParams string `json:"queueParams,omitempty"` + Vhost string `json:"vhost"` + // QueueName string `json:"queueName,omitempty"` + // QueueParams string `json:"queueParams,omitempty"` } diff --git a/api/v1alpha1/testrun_webhook.go b/api/v1alpha1/testrun_webhook.go index 7109710b..1ad03700 100644 --- a/api/v1alpha1/testrun_webhook.go +++ b/api/v1alpha1/testrun_webhook.go @@ -70,7 +70,7 @@ func (r *TestRun) Default() { r.Spec.SuiteRunner = &SuiteRunner{&cluster.Spec.ETOS.SuiteRunner.Image} } if r.Spec.LogListener == nil && cluster != nil { - r.Spec.LogListener = &LogListener{&cluster.Spec.ETOS.SuiteRunner.LogListener} + r.Spec.LogListener = &LogListener{&cluster.Spec.ETOS.SuiteRunner.LogListener.Image} } if r.Spec.EnvironmentProvider == nil && cluster != nil { r.Spec.EnvironmentProvider = &EnvironmentProvider{&cluster.Spec.ETOS.EnvironmentProvider.Image} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 69fe4fff..ccf76512 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -151,6 +151,7 @@ func (in *ETOS) DeepCopyInto(out *ETOS) { out.LogArea = in.LogArea out.SuiteRunner = in.SuiteRunner out.TestRunner = in.TestRunner + out.SuiteStarter = in.SuiteStarter out.EnvironmentProvider = in.EnvironmentProvider in.Ingress.DeepCopyInto(&out.Ingress) in.Config.DeepCopyInto(&out.Config) @@ -231,6 +232,22 @@ func (in *ETOSLogArea) DeepCopy() *ETOSLogArea { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ETOSLogListener) DeepCopyInto(out *ETOSLogListener) { + *out = *in + out.Image = in.Image +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETOSLogListener. +func (in *ETOSLogListener) DeepCopy() *ETOSLogListener { + if in == nil { + return nil + } + out := new(ETOSLogListener) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ETOSSSE) DeepCopyInto(out *ETOSSSE) { *out = *in @@ -264,6 +281,38 @@ func (in *ETOSSuiteRunner) DeepCopy() *ETOSSuiteRunner { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ETOSSuiteStarter) DeepCopyInto(out *ETOSSuiteStarter) { + *out = *in + out.Image = in.Image + out.Config = in.Config +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETOSSuiteStarter. +func (in *ETOSSuiteStarter) DeepCopy() *ETOSSuiteStarter { + if in == nil { + return nil + } + out := new(ETOSSuiteStarter) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ETOSSuiteStarterConfig) DeepCopyInto(out *ETOSSuiteStarterConfig) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ETOSSuiteStarterConfig. +func (in *ETOSSuiteStarterConfig) DeepCopy() *ETOSSuiteStarterConfig { + if in == nil { + return nil + } + out := new(ETOSSuiteStarterConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ETOSTestRunner) DeepCopyInto(out *ETOSTestRunner) { *out = *in diff --git a/config/crd/bases/etos.eiffel-community.github.io_clusters.yaml b/config/crd/bases/etos.eiffel-community.github.io_clusters.yaml index c748c175..763dc0be 100644 --- a/config/crd/bases/etos.eiffel-community.github.io_clusters.yaml +++ b/config/crd/bases/etos.eiffel-community.github.io_clusters.yaml @@ -48,12 +48,14 @@ spec: properties: database: default: {} + description: Database describes the deployment of a database for ETOS. properties: deploy: default: true type: boolean etcd: default: {} + description: Etcd describes the deployment of an ETCD database. properties: host: default: etcd-client @@ -67,11 +69,15 @@ spec: type: object etos: default: {} + description: ETOS describes the deployment of an ETOS cluster. properties: api: default: image: registry.nordix.org/eiffel/etos-api:latest + description: ETOSAPI describes the deployment of the ETOS API. properties: + executionSpaceProviderSecret: + type: string image: type: string imagePullPolicy: @@ -79,6 +85,10 @@ spec: description: PullPolicy describes a policy for if/when to pull a container image type: string + iutProviderSecret: + type: string + logAreaProviderSecret: + type: string required: - image type: object @@ -86,6 +96,7 @@ spec: default: encryptionKey: value: "" + description: ETOSConfig describes a common configuration for ETOS. properties: dev: default: "true" @@ -176,6 +187,8 @@ spec: default: "10" type: string testrunRetention: + description: Retention describes the failure and success retentions + for testruns. properties: failure: type: string @@ -190,6 +203,8 @@ spec: environmentProvider: default: image: registry.nordix.org/eiffel/etos-environment-provider:latest + description: ETOSEnvironmentProvider describes the deployment + of an ETOS environment provider. properties: image: type: string @@ -220,6 +235,8 @@ spec: logArea: default: image: registry.nordix.org/eiffel/etos-log-area:latest + description: ETOSLogArea describes th deployment of an ETOS log + area API. properties: image: type: string @@ -234,6 +251,7 @@ spec: sse: default: image: registry.nordix.org/eiffel/etos-sse:latest + description: ETOSSSE describes th deployment of an ETOS SSE API. properties: image: type: string @@ -250,6 +268,8 @@ spec: image: registry.nordix.org/eiffel/etos-suite-runner:latest logListener: image: registry.nordix.org/eiffel/etos-log-listener:latest + description: ETOSSuiteRunner describes the deployment of an ETOS + suite runner. properties: image: type: string @@ -259,8 +279,15 @@ spec: pull a container image type: string logListener: - description: Image configuration. + description: ETOSLogListener describes the deployment of an + ETOS log listener. properties: + etosQueueName: + default: etos-*-temp + type: string + etosQueueParams: + default: "" + type: string image: type: string imagePullPolicy: @@ -275,9 +302,62 @@ spec: - image - logListener type: object + suiteStarter: + default: + image: registry.nordix.org/eiffel/etos-suite-starter:latest + description: ETOSSuiteStarter describes the deployment of an ETOS + suite starter. + properties: + config: + default: + gracePeriod: "300" + ttl: "3600" + description: |- + ETOSSuiteStarterConfig describes the configuration required for a suite starter. + This is separate from the ETOSConfig as we want to remove this in the future when the suite + starter is no longer in use. + properties: + gracePeriod: + default: "300" + type: string + observabilityConfigmapName: + default: "" + type: string + otelCollectorHost: + default: "" + type: string + sidecarImage: + default: "" + type: string + ttl: + default: "3600" + type: string + type: object + eiffelQueueName: + default: etos-suite-starter + type: string + eiffelQueueParams: + default: "" + type: string + image: + type: string + imagePullPolicy: + default: IfNotPresent + description: PullPolicy describes a policy for if/when to + pull a container image + type: string + suiteRunnerTemplateConfigmapName: + default: "" + description: Provide a custom suite runner template. + type: string + required: + - image + type: object testRunner: default: version: latest + description: ETOSTestRunner describes the deployment of an ETOS + test runner. properties: version: type: string @@ -287,6 +367,8 @@ spec: type: object eventRepository: default: {} + description: EventRepository describes the deployment of an event + repository for ETOS. properties: api: default: @@ -309,6 +391,12 @@ spec: default: false description: Deploy a local event repository for a cluster. type: boolean + eiffelQueueName: + default: etos + type: string + eiffelQueueParams: + default: "" + type: string host: default: eventrepository type: string @@ -331,6 +419,7 @@ spec: type: object mongo: default: {} + description: MongoDB describes the deployment of a MongoDB. properties: deploy: default: false @@ -421,10 +510,10 @@ spec: type: object messageBus: default: {} + description: MessageBus describes the deployment of messagesbuses + for ETOS. properties: eiffel: - default: - queueName: etos description: RabbitMQ configuration. properties: deploy: @@ -505,10 +594,6 @@ spec: port: default: "5672" type: string - queueName: - type: string - queueParams: - type: string ssl: default: "false" type: string @@ -520,8 +605,6 @@ spec: type: string type: object logs: - default: - queueName: etos-*-temp description: RabbitMQ configuration. properties: deploy: @@ -602,10 +685,6 @@ spec: port: default: "5672" type: string - queueName: - type: string - queueParams: - type: string ssl: default: "false" type: string diff --git a/config/crd/bases/etos.eiffel-community.github.io_environmentrequests.yaml b/config/crd/bases/etos.eiffel-community.github.io_environmentrequests.yaml index bae8329c..7f09f4e4 100644 --- a/config/crd/bases/etos.eiffel-community.github.io_environmentrequests.yaml +++ b/config/crd/bases/etos.eiffel-community.github.io_environmentrequests.yaml @@ -113,7 +113,7 @@ spec: description: TestEnvironment to run tests within. type: object execution: - description: Execution describes hot to execute a testCase. + description: Execution describes how to execute a testCase. properties: checkout: items: diff --git a/config/crd/bases/etos.eiffel-community.github.io_environments.yaml b/config/crd/bases/etos.eiffel-community.github.io_environments.yaml index 1b9aed99..e3ca901c 100644 --- a/config/crd/bases/etos.eiffel-community.github.io_environments.yaml +++ b/config/crd/bases/etos.eiffel-community.github.io_environments.yaml @@ -69,7 +69,7 @@ spec: description: TestEnvironment to run tests within. type: object execution: - description: Execution describes hot to execute a testCase. + description: Execution describes how to execute a testCase. properties: checkout: items: diff --git a/config/crd/bases/etos.eiffel-community.github.io_providers.yaml b/config/crd/bases/etos.eiffel-community.github.io_providers.yaml index feebb6b6..64018ffe 100644 --- a/config/crd/bases/etos.eiffel-community.github.io_providers.yaml +++ b/config/crd/bases/etos.eiffel-community.github.io_providers.yaml @@ -125,7 +125,7 @@ spec: for an IUT. properties: stages: - description: JSONTasIUTPrepareaStages defines the preparation + description: JSONTasIUTPrepareStages defines the preparation stages required for an IUT. properties: environment_provider: diff --git a/config/crd/bases/etos.eiffel-community.github.io_testruns.yaml b/config/crd/bases/etos.eiffel-community.github.io_testruns.yaml index 3e06699a..967e87f7 100644 --- a/config/crd/bases/etos.eiffel-community.github.io_testruns.yaml +++ b/config/crd/bases/etos.eiffel-community.github.io_testruns.yaml @@ -116,6 +116,8 @@ spec: - logArea type: object retention: + description: Retention describes the failure and success retentions + for testruns. properties: failure: type: string @@ -149,14 +151,14 @@ spec: description: Priority to execute the test suite. type: integer tests: - description: Tests to execute as part of this testrun. + description: Tests to execute as part of this suite. items: properties: environment: description: TestEnvironment to run tests within. type: object execution: - description: Execution describes hot to execute a testCase. + description: Execution describes how to execute a testCase. properties: checkout: items: diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 0af84fcb..9c540e63 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -2,3 +2,7 @@ resources: - manager.yaml apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization +images: +- name: controller + newName: registry.nordix.org/eiffel/etos-controller + newTag: latest diff --git a/internal/controller/environmentrequest_controller.go b/internal/controller/environmentrequest_controller.go index f8370c92..b40de743 100644 --- a/internal/controller/environmentrequest_controller.go +++ b/internal/controller/environmentrequest_controller.go @@ -116,7 +116,7 @@ func (r *EnvironmentRequestReconciler) reconcile(ctx context.Context, environmen logger.V(1).Info("environment provider count", "active", len(environmentProviders.activeJobs), "successful", len(environmentProviders.successfulJobs), "failed", len(environmentProviders.failedJobs)) // Check providers availability - // TODO Update status + // TODO: Update status providers := etosv1alpha1.Providers{ IUT: environmentrequest.Spec.Providers.IUT.ID, ExecutionSpace: environmentrequest.Spec.Providers.ExecutionSpace.ID, diff --git a/internal/etos/api/api.go b/internal/etos/api/api.go index 055732f2..0c5c56eb 100644 --- a/internal/etos/api/api.go +++ b/internal/etos/api/api.go @@ -322,6 +322,7 @@ func (r *ETOSApiDeployment) deployment(name types.NamespacedName) *appsv1.Deploy Spec: corev1.PodSpec{ ServiceAccountName: name.Name, Containers: []corev1.Container{r.container(name)}, + Volumes: r.volumes(), }, }, }, @@ -379,14 +380,67 @@ func (r *ETOSApiDeployment) container(name types.NamespacedName) corev1.Containe Protocol: "TCP", }, }, + VolumeMounts: r.volumeMounts(), LivenessProbe: probe, ReadinessProbe: probe, - EnvFrom: r.environment(), + EnvFrom: r.environmentFrom(), + Env: r.environment(), } } -// environment creates the environment resource for the ETOS API deployment. -func (r *ETOSApiDeployment) environment() []corev1.EnvFromSource { +// volumeMounts creates the container volume mounts for providers if necessary. +func (r *ETOSApiDeployment) volumeMounts() []corev1.VolumeMount { + mounts := []corev1.VolumeMount{} + if r.IUTProviderSecret != "" { + mounts = append(mounts, corev1.VolumeMount{Name: "iut-providers", ReadOnly: true, MountPath: "/providers/iut"}) + } + if r.LogAreaProviderSecret != "" { + mounts = append(mounts, corev1.VolumeMount{Name: "log-area-providers", ReadOnly: true, MountPath: "/providers/log_area"}) + } + if r.ExecutionSpaceProviderSecret != "" { + mounts = append(mounts, corev1.VolumeMount{Name: "execution-space-providers", ReadOnly: true, MountPath: "/providers/execution_space"}) + } + return mounts +} + +// volumes creates the volume specification for the ETOS API pod. +func (r *ETOSApiDeployment) volumes() []corev1.Volume { + vol := []corev1.Volume{} + if r.IUTProviderSecret != "" { + vol = append(vol, corev1.Volume{ + Name: "iut-providers", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: r.IUTProviderSecret, + }, + }, + }) + } + if r.LogAreaProviderSecret != "" { + vol = append(vol, corev1.Volume{ + Name: "log-area-providers", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: r.LogAreaProviderSecret, + }, + }, + }) + } + if r.ExecutionSpaceProviderSecret != "" { + vol = append(vol, corev1.Volume{ + Name: "execution-space-providers", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: r.ExecutionSpaceProviderSecret, + }, + }, + }) + } + return vol +} + +// environmentFrom creates the environment resource for the ETOS API deployment. +func (r *ETOSApiDeployment) environmentFrom() []corev1.EnvFromSource { return []corev1.EnvFromSource{ { SecretRef: &corev1.SecretEnvSource{ @@ -412,6 +466,21 @@ func (r *ETOSApiDeployment) environment() []corev1.EnvFromSource { } } +// environment creates environment variables for providers if supplied. +func (r *ETOSApiDeployment) environment() []corev1.EnvVar { + env := []corev1.EnvVar{} + if r.IUTProviderSecret != "" { + env = append(env, corev1.EnvVar{Name: "IUT_PROVIDERS", Value: "/providers/iut"}) + } + if r.LogAreaProviderSecret != "" { + env = append(env, corev1.EnvVar{Name: "LOG_AREA_PROVIDERS", Value: "/providers/log_area"}) + } + if r.ExecutionSpaceProviderSecret != "" { + env = append(env, corev1.EnvVar{Name: "EXECUTION_SPACE_PROVIDERS", Value: "/providers/execution_space"}) + } + return env +} + // meta creates the common meta resource for the ETOS API deployment. func (r *ETOSApiDeployment) meta(name types.NamespacedName) metav1.ObjectMeta { return metav1.ObjectMeta{ diff --git a/internal/etos/etos.go b/internal/etos/etos.go index a16e5d38..e0e39461 100644 --- a/internal/etos/etos.go +++ b/internal/etos/etos.go @@ -22,6 +22,7 @@ import ( etosv1alpha1 "github.com/eiffel-community/etos/api/v1alpha1" etosapi "github.com/eiffel-community/etos/internal/etos/api" + etossuitestarter "github.com/eiffel-community/etos/internal/etos/suitestarter" corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -76,7 +77,7 @@ func (r *ETOSDeployment) Reconcile(ctx context.Context, cluster *etosv1alpha1.Cl return err } - _, err = r.reconcileSecret(ctx, namespacedName, cluster) + encryption, err := r.reconcileSecret(ctx, namespacedName, cluster) if err != nil { return err } @@ -96,6 +97,11 @@ func (r *ETOSDeployment) Reconcile(ctx context.Context, cluster *etosv1alpha1.Cl return err } + suitestarter := etossuitestarter.NewETOSSuiteStarterDeployment(r.SuiteStarter, r.Scheme, r.Client, r.rabbitmqSecret, r.messagebusSecret, configmap, encryption) + if err := suitestarter.Reconcile(ctx, cluster); err != nil { + return err + } + return nil } @@ -282,7 +288,7 @@ func (r *ETOSDeployment) configmap(name types.NamespacedName, cluster *etosv1alp "ETOS_API": etosApi, "SUITE_RUNNER_IMAGE": cluster.Spec.ETOS.SuiteRunner.Image.Image, "SUITE_RUNNER_IMAGE_PULL_POLICY": string(cluster.Spec.ETOS.SuiteRunner.ImagePullPolicy), - "LOG_LISTENER_IMAGE": cluster.Spec.ETOS.SuiteRunner.LogListener.Image, + "LOG_LISTENER_IMAGE": cluster.Spec.ETOS.SuiteRunner.LogListener.Image.Image, "LOG_LISTENER_IMAGE_PULL_POLICY": string(cluster.Spec.ETOS.SuiteRunner.LogListener.ImagePullPolicy), "ENVIRONMENT_PROVIDER_IMAGE": cluster.Spec.ETOS.EnvironmentProvider.Image.Image, "ENVIRONMENT_PROVIDER_IMAGE_PULL_POLICY": string(cluster.Spec.ETOS.EnvironmentProvider.ImagePullPolicy), diff --git a/internal/etos/suitestarter/suitestarter.go b/internal/etos/suitestarter/suitestarter.go new file mode 100644 index 00000000..5958fe93 --- /dev/null +++ b/internal/etos/suitestarter/suitestarter.go @@ -0,0 +1,635 @@ +// Copyright Axis Communications AB. +// +// For a full list of individual contributors, please see the commit history. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package suitestarter + +import ( + "context" + "fmt" + "maps" + + etosv1alpha1 "github.com/eiffel-community/etos/api/v1alpha1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type ETOSSuiteStarterDeployment struct { + etosv1alpha1.ETOSSuiteStarter + client.Client + Scheme *runtime.Scheme + rabbitmqSecret string + messagebusSecret string + etosConfigmap *corev1.ConfigMap + encryptionSecret *corev1.Secret +} + +// NewETOSSuiteStarterDeployment will create a new ETOS SuiteStarter reconciler. +func NewETOSSuiteStarterDeployment(spec etosv1alpha1.ETOSSuiteStarter, scheme *runtime.Scheme, client client.Client, rabbitmqSecret, messagebusSecret string, configmap *corev1.ConfigMap, encryption *corev1.Secret) *ETOSSuiteStarterDeployment { + return &ETOSSuiteStarterDeployment{spec, client, scheme, rabbitmqSecret, messagebusSecret, configmap, encryption} +} + +// Reconcile will reconcile the ETOS suite starter to its expected state. +func (r *ETOSSuiteStarterDeployment) Reconcile(ctx context.Context, cluster *etosv1alpha1.Cluster) error { + var err error + name := fmt.Sprintf("%s-etos-suite-starter", cluster.Name) + namespacedName := types.NamespacedName{Name: name, Namespace: cluster.Namespace} + // There is an assumption in the suite starter that the service account is named 'etos-sa'. + _, err = r.reconcileSuiteRunnerServiceAccount(ctx, types.NamespacedName{Name: "etos-sa", Namespace: cluster.Namespace}, cluster) + if err != nil { + return err + } + secret, err := r.reconcileSuiteRunnerSecret(ctx, namespacedName, cluster) + if err != nil { + return err + } + configmap, err := r.reconcileConfigmap(ctx, secret.ObjectMeta.Name, namespacedName, cluster) + if err != nil { + return err + } + + template, err := r.reconcileTemplate(ctx, namespacedName, cluster) + if err != nil { + return err + } + var suiteRunnerTemplateName string + if r.SuiteRunnerTemplateConfigmapName == "" { + suiteRunnerTemplateName = template.ObjectMeta.Name + } else { + suiteRunnerTemplateName = r.SuiteRunnerTemplateConfigmapName + } + _, err = r.reconcileDeployment(ctx, configmap.ObjectMeta.Name, suiteRunnerTemplateName, namespacedName, cluster) + if err != nil { + return err + } + _, err = r.reconcileSecret(ctx, namespacedName, cluster) + if err != nil { + return err + } + _, err = r.reconcileRole(ctx, namespacedName, cluster) + if err != nil { + return err + } + _, err = r.reconcileServiceAccount(ctx, namespacedName, cluster) + if err != nil { + return err + } + _, err = r.reconcileRolebinding(ctx, namespacedName, cluster) + if err != nil { + return err + } + return err +} + +// reconcileSuiteRunnerServiceAccount will reconcile the ETOS SuiteStarter service account to its expected state. +func (r *ETOSSuiteStarterDeployment) reconcileSuiteRunnerServiceAccount(ctx context.Context, name types.NamespacedName, owner metav1.Object) (*corev1.ServiceAccount, error) { + target := r.serviceaccount(name) + if err := ctrl.SetControllerReference(owner, target, r.Scheme); err != nil { + return target, err + } + + serviceaccount := &corev1.ServiceAccount{} + if err := r.Get(ctx, name, serviceaccount); err != nil { + if !apierrors.IsNotFound(err) { + return serviceaccount, err + } + if err := r.Create(ctx, target); err != nil { + return target, err + } + return target, nil + } + return target, r.Patch(ctx, target, client.StrategicMergeFrom(serviceaccount)) +} + +// reconcileSuiteRunnerSecret will reconcile the ETOS suite runner secret to its expected state. +func (r *ETOSSuiteStarterDeployment) reconcileSuiteRunnerSecret(ctx context.Context, name types.NamespacedName, owner metav1.Object) (*corev1.Secret, error) { + name.Name = fmt.Sprintf("%s-suite-runner", name.Name) + target, err := r.mergedSecret(ctx, name) + if err != nil { + return nil, err + } + if err := ctrl.SetControllerReference(owner, target, r.Scheme); err != nil { + return target, err + } + + secret := &corev1.Secret{} + if err := r.Get(ctx, name, secret); err != nil { + if !apierrors.IsNotFound(err) { + return secret, err + } + if err := r.Create(ctx, target); err != nil { + return target, err + } + return target, nil + } + return target, r.Patch(ctx, target, client.StrategicMergeFrom(secret)) +} + +// reconcileConfigmap will reconcile the ETOS suite starter config to its expected state. +func (r *ETOSSuiteStarterDeployment) reconcileConfigmap(ctx context.Context, secretName string, name types.NamespacedName, cluster *etosv1alpha1.Cluster) (*corev1.ConfigMap, error) { + target := r.configmap(name, secretName, cluster) + if err := ctrl.SetControllerReference(cluster, target, r.Scheme); err != nil { + return target, err + } + + configmap := &corev1.ConfigMap{} + if err := r.Get(ctx, name, configmap); err != nil { + if !apierrors.IsNotFound(err) { + return configmap, err + } + if err := r.Create(ctx, target); err != nil { + return target, err + } + return target, nil + } + return target, r.Patch(ctx, target, client.StrategicMergeFrom(configmap)) +} + +// reconcileTemplate will reconcile the ETOS SuiteRunner template to its expected state. +func (r *ETOSSuiteStarterDeployment) reconcileTemplate(ctx context.Context, name types.NamespacedName, owner metav1.Object) (*corev1.ConfigMap, error) { + name.Name = fmt.Sprintf("%s-template", name.Name) + target := r.suiteRunnerTemplate(name) + if err := ctrl.SetControllerReference(owner, target, r.Scheme); err != nil { + return target, err + } + scheme.Scheme.Default(target) + + configmap := &corev1.ConfigMap{} + if err := r.Get(ctx, name, configmap); err != nil { + if !apierrors.IsNotFound(err) { + return configmap, err + } + if err := r.Create(ctx, target); err != nil { + return target, err + } + return target, nil + } + if equality.Semantic.DeepDerivative(target.Data, configmap.Data) { + return configmap, nil + } + return target, r.Patch(ctx, target, client.StrategicMergeFrom(configmap)) +} + +// reconcileDeployment will reconcile the ETOS SuiteStarter deployment to its expected state. +func (r *ETOSSuiteStarterDeployment) reconcileDeployment(ctx context.Context, configmap string, suiteRunnerTemplate string, name types.NamespacedName, owner metav1.Object) (*appsv1.Deployment, error) { + target := r.deployment(name, configmap, suiteRunnerTemplate) + if err := ctrl.SetControllerReference(owner, target, r.Scheme); err != nil { + return target, err + } + scheme.Scheme.Default(target) + + deployment := &appsv1.Deployment{} + if err := r.Get(ctx, name, deployment); err != nil { + if !apierrors.IsNotFound(err) { + return deployment, err + } + if err := r.Create(ctx, target); err != nil { + return target, err + } + return target, nil + } + if equality.Semantic.DeepDerivative(target.Spec, deployment.Spec) { + return deployment, nil + } + return target, r.Patch(ctx, target, client.StrategicMergeFrom(deployment)) +} + +// reconcileSecret will reconcile the ETOS SuiteStarter service account secret to its expected state. +func (r *ETOSSuiteStarterDeployment) reconcileSecret(ctx context.Context, name types.NamespacedName, owner metav1.Object) (*corev1.Secret, error) { + target := r.secret(name) + if err := ctrl.SetControllerReference(owner, target, r.Scheme); err != nil { + return target, err + } + scheme.Scheme.Default(target) + + secret := &corev1.Secret{} + if err := r.Get(ctx, name, secret); err != nil { + if !apierrors.IsNotFound(err) { + return secret, err + } + if err := r.Create(ctx, target); err != nil { + return target, err + } + return target, nil + } + if equality.Semantic.DeepDerivative(target.Data, secret.Data) { + return secret, nil + } + return target, r.Patch(ctx, target, client.StrategicMergeFrom(secret)) +} + +// reconcileRole will reconcile the ETOS SuiteStarter service account role to its expected state. +func (r *ETOSSuiteStarterDeployment) reconcileRole(ctx context.Context, name types.NamespacedName, owner metav1.Object) (*rbacv1.Role, error) { + labelName := name.Name + name.Name = fmt.Sprintf("%s:sa:esr-handler", name.Name) + + target := r.role(name, labelName) + if err := ctrl.SetControllerReference(owner, target, r.Scheme); err != nil { + return target, err + } + + role := &rbacv1.Role{} + if err := r.Get(ctx, name, role); err != nil { + if !apierrors.IsNotFound(err) { + return role, err + } + if err := r.Create(ctx, target); err != nil { + return target, err + } + return target, nil + } + return target, r.Patch(ctx, target, client.StrategicMergeFrom(role)) +} + +// reconcileServiceAccount will reconcile the ETOS SuiteStarter service account to its expected state. +func (r *ETOSSuiteStarterDeployment) reconcileServiceAccount(ctx context.Context, name types.NamespacedName, owner metav1.Object) (*corev1.ServiceAccount, error) { + target := r.serviceaccount(name) + if err := ctrl.SetControllerReference(owner, target, r.Scheme); err != nil { + return target, err + } + + serviceaccount := &corev1.ServiceAccount{} + if err := r.Get(ctx, name, serviceaccount); err != nil { + if !apierrors.IsNotFound(err) { + return serviceaccount, err + } + if err := r.Create(ctx, target); err != nil { + return target, err + } + return target, nil + } + return target, r.Patch(ctx, target, client.StrategicMergeFrom(serviceaccount)) +} + +// reconcileRolebinding will reconcile the ETOS SuiteStarter service account role binding to its expected state. +func (r *ETOSSuiteStarterDeployment) reconcileRolebinding(ctx context.Context, name types.NamespacedName, owner metav1.Object) (*rbacv1.RoleBinding, error) { + target := r.rolebinding(name) + if err := ctrl.SetControllerReference(owner, target, r.Scheme); err != nil { + return target, err + } + + rolebinding := &rbacv1.RoleBinding{} + if err := r.Get(ctx, name, rolebinding); err != nil { + if !apierrors.IsNotFound(err) { + return rolebinding, err + } + if err := r.Create(ctx, target); err != nil { + return target, err + } + return target, nil + } + return target, r.Patch(ctx, target, client.StrategicMergeFrom(rolebinding)) +} + +// configmap creates a configmap resource definition for the ETOS suite runner. +func (r *ETOSSuiteStarterDeployment) configmap(name types.NamespacedName, secretName string, cluster *etosv1alpha1.Cluster) *corev1.ConfigMap { + data := map[string]string{ + "SUITE_RUNNER": cluster.Spec.ETOS.SuiteRunner.Image.Image, + "LOG_LISTENER": cluster.Spec.ETOS.SuiteRunner.LogListener.Image.Image, + "ETOS_CONFIGMAP": r.etosConfigmap.ObjectMeta.Name, + "ETOS_RABBITMQ_SECRET": secretName, + "ETOS_ESR_TTL": r.Config.TTL, + "ETOS_TERMINATION_GRACE_PERIOD": r.Config.GracePeriod, + } + if r.Config.ObservabilityConfigmapName != "" { + data["ETOS_OBSERVABILITY_CONFIGMAP"] = r.Config.ObservabilityConfigmapName + } + if r.Config.SidecarImage != "" { + data["ETOS_SIDECAR_IMAGE"] = r.Config.SidecarImage + } + if r.Config.OTELCollectorHost != "" { + data["OTEL_COLLECTOR_HOST"] = r.Config.OTELCollectorHost + } + maps.Copy(data, r.etosConfigmap.Data) + return &corev1.ConfigMap{ + ObjectMeta: r.meta(name), + Data: data, + } +} + +// secret creates a secret resource definition for the ETOS SuiteStarter. +func (r *ETOSSuiteStarterDeployment) secret(name types.NamespacedName) *corev1.Secret { + meta := r.meta(name) + meta.Annotations["kubernetes.io/service-account.name"] = name.Name + name.Name = fmt.Sprintf("%s-token", name.Name) + return &corev1.Secret{ + ObjectMeta: meta, + Type: corev1.SecretTypeServiceAccountToken, + } +} + +// mergedSecret creates a secret that is the merged values of Eiffel, ETOS and encryption key secrets. +// This is for use in the suite runner which only has a single secret as input. +func (r *ETOSSuiteStarterDeployment) mergedSecret(ctx context.Context, name types.NamespacedName) (*corev1.Secret, error) { + eiffel := &corev1.Secret{} + if err := r.Get(ctx, types.NamespacedName{Name: r.rabbitmqSecret, Namespace: name.Namespace}, eiffel); err != nil { + return nil, err + } + etos := &corev1.Secret{} + if err := r.Get(ctx, types.NamespacedName{Name: r.messagebusSecret, Namespace: name.Namespace}, etos); err != nil { + return nil, err + } + data := map[string][]byte{} + maps.Copy(data, eiffel.Data) + maps.Copy(data, etos.Data) + maps.Copy(data, r.encryptionSecret.Data) + return &corev1.Secret{ + ObjectMeta: r.meta(name), + Data: data, + }, nil +} + +// role creates a role resource definition for the ETOS SuiteStarter. +func (r *ETOSSuiteStarterDeployment) role(name types.NamespacedName, labelName string) *rbacv1.Role { + meta := r.meta(types.NamespacedName{Name: labelName, Namespace: name.Namespace}) + meta.Name = name.Name + meta.Annotations["rbac.authorization.kubernetes.io/autoupdate"] = "true" + return &rbacv1.Role{ + ObjectMeta: meta, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{ + "batch", + }, + Resources: []string{ + "jobs", + }, + Verbs: []string{ + "get", "create", "delete", "list", "watch", + }, + }, + { + APIGroups: []string{""}, + Resources: []string{ + "pods", + }, + Verbs: []string{ + "get", "list", "watch", + }, + }, + }, + } +} + +// serviceaccount creates a service account resource definition for the ETOS SuiteStarter. +func (r *ETOSSuiteStarterDeployment) serviceaccount(name types.NamespacedName) *corev1.ServiceAccount { + return &corev1.ServiceAccount{ + ObjectMeta: r.meta(name), + } +} + +// rolebinding creates a rolebinding resource definition for the ETOS SuiteStarter. +func (r *ETOSSuiteStarterDeployment) rolebinding(name types.NamespacedName) *rbacv1.RoleBinding { + return &rbacv1.RoleBinding{ + ObjectMeta: r.meta(name), + RoleRef: rbacv1.RoleRef{ + APIGroup: rbacv1.SchemeGroupVersion.Group, + Kind: "Role", + Name: fmt.Sprintf("%s:sa:esr-handler", name.Name), + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: name.Name, + }, + }, + } +} + +// deployment creates a deployment resource definition for the ETOS SuiteStarter. +func (r *ETOSSuiteStarterDeployment) deployment(name types.NamespacedName, configmap string, suiteRunnerTemplate string) *appsv1.Deployment { + return &appsv1.Deployment{ + ObjectMeta: r.meta(name), + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app.kubernetes.io/name": name.Name, + "app.kubernetes.io/part-of": "etos", + "app.kubernetes.io/component": "suitestarter", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: r.meta(name), + Spec: corev1.PodSpec{ + ServiceAccountName: name.Name, + Containers: []corev1.Container{r.container(name, configmap)}, + Volumes: []corev1.Volume{{ + Name: "suite-runner-template", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: suiteRunnerTemplate, + }, + }, + }, + }}, + }, + }, + }, + } +} + +// container creates the container resource for the ETOS SuiteStarter deployment. +func (r *ETOSSuiteStarterDeployment) container(name types.NamespacedName, configmap string) corev1.Container { + env := []corev1.EnvVar{ + {Name: "RABBITMQ_QUEUE", Value: r.EiffelQueueName}, + } + if r.EiffelQueueParams != "" { + env = append(env, corev1.EnvVar{Name: "RABBITMQ_QUEUE_PARAMS", Value: r.EiffelQueueParams}) + } + return corev1.Container{ + Name: name.Name, + Image: r.Image.Image, + ImagePullPolicy: r.ImagePullPolicy, + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("256Mi"), + corev1.ResourceCPU: resource.MustParse("200m"), + }, + Requests: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("128Mi"), + corev1.ResourceCPU: resource.MustParse("100m"), + }, + }, + EnvFrom: r.environment(configmap), + Env: env, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "suite-runner-template", + MountPath: "/app/suite_runner_template.yaml", + SubPath: "suite_runner_template.yaml", + }, + }, + } +} + +// suiteRunnerTemplate creates a configmap resource for the ETOS SuiteStarter. +func (r *ETOSSuiteStarterDeployment) suiteRunnerTemplate(name types.NamespacedName) *corev1.ConfigMap { + data := map[string]string{ + "suite_runner_template.yaml": ` + apiVersion: batch/v1 + kind: Job + metadata: + name: {job_name} + labels: + app: suite-runner + id: {suite_id} + spec: + ttlSecondsAfterFinished: {ttl} + template: + metadata: + name: {job_name} + spec: + terminationGracePeriodSeconds: {termination_grace_period} + volumes: + - name: kubexit + emptyDir: {{}} + - name: graveyard + emptyDir: + medium: Memory + initContainers: + - name: kubexit + image: karlkfi/kubexit:latest + command: ["cp", "/bin/kubexit", "/kubexit/kubexit"] + volumeMounts: + - mountPath: /kubexit + name: kubexit + - name: create-queue + image: registry.nordix.org/eiffel/etos-log-listener:4969c9b2 + command: ["python", "-u", "-m", "create_queue"] + envFrom: + - configMapRef: + name: {etos_configmap} + - secretRef: + name: {etos_rabbitmq_secret} + env: + - name: TERCC + value: '{EiffelTestExecutionRecipeCollectionCreatedEvent}' + serviceAccountName: etos-sa + containers: + - name: {job_name} + image: {docker_image} + imagePullPolicy: Always + command: ['/kubexit/kubexit'] + args: ['python', '-u', '-m', 'etos_suite_runner'] + envFrom: + - configMapRef: + name: {etos_configmap} + - secretRef: + name: {etos_rabbitmq_secret} + - configMapRef: + name: {etos_observability_configmap} + env: + - name: TERCC + value: '{EiffelTestExecutionRecipeCollectionCreatedEvent}' + - name: KUBEXIT_NAME + value: esr + - name: KUBEXIT_GRAVEYARD + value: /graveyard + - name: OTEL_CONTEXT + value: {otel_context} + - name: OTEL_COLLECTOR_HOST + value: {otel_collector_host} + volumeMounts: + - name: graveyard + mountPath: /graveyard + - name: kubexit + mountPath: /kubexit + - name: etos-log-listener + image: {log_listener} + imagePullPolicy: Always + command: ['/kubexit/kubexit'] + args: ['python', '-u', '-m', 'log_listener'] + envFrom: + - configMapRef: + name: {etos_configmap} + - secretRef: + name: {etos_rabbitmq_secret} + env: + - name: TERCC + value: '{EiffelTestExecutionRecipeCollectionCreatedEvent}' + - name: KUBEXIT_NAME + value: log_listener + - name: KUBEXIT_GRACE_PERIOD + value: '400s' # needs to be greater than grace period of the container + - name: KUBEXIT_GRAVEYARD + value: /graveyard + - name: KUBEXIT_DEATH_DEPS + value: esr + volumeMounts: + - name: graveyard + mountPath: /graveyard + - name: kubexit + mountPath: /kubexit + restartPolicy: Never + backoffLimit: 0 + `, + } + return &corev1.ConfigMap{ + ObjectMeta: r.meta(name), + Data: data, + } +} + +// environment creates the environment resource for the ETOS SuiteStarter deployment. +func (r *ETOSSuiteStarterDeployment) environment(configmap string) []corev1.EnvFromSource { + return []corev1.EnvFromSource{ + { + SecretRef: &corev1.SecretEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: r.rabbitmqSecret, + }, + }, + }, + { + SecretRef: &corev1.SecretEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: r.messagebusSecret, + }, + }, + }, + { + ConfigMapRef: &corev1.ConfigMapEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: configmap, + }, + }, + }, + } +} + +// meta creates the common meta resource for the ETOS SuiteStarter deployment. +func (r *ETOSSuiteStarterDeployment) meta(name types.NamespacedName) metav1.ObjectMeta { + return metav1.ObjectMeta{ + Labels: map[string]string{ + "app.kubernetes.io/name": name.Name, + "app.kubernetes.io/part-of": "etos", + "app.kubernetes.io/component": "suitestarter", + }, + Annotations: make(map[string]string), + Name: name.Name, + Namespace: name.Namespace, + } +} diff --git a/internal/extras/eventrepository.go b/internal/extras/eventrepository.go index 1884ac0d..ca5ed05c 100644 --- a/internal/extras/eventrepository.go +++ b/internal/extras/eventrepository.go @@ -238,6 +238,10 @@ func (r *EventRepositoryDeployment) containers(name types.NamespacedName) []core "eiffel_graphql_api.storage", }, EnvFrom: r.environment(), + Env: []corev1.EnvVar{ + {Name: "RABBITMQ_QUEUE", Value: r.EventRepository.EiffelQueueName}, + {Name: "RABBITMQ_QUEUE_PARAMS", Value: r.EventRepository.EiffelQueueParams}, + }, }, } } diff --git a/internal/extras/messagebus.go b/internal/extras/messagebus.go index 12809f8c..2773527a 100644 --- a/internal/extras/messagebus.go +++ b/internal/extras/messagebus.go @@ -214,12 +214,6 @@ func (r *MessageBusDeployment) secretData(namespace string) (map[string][]byte, if r.Username != "" { data["ETOS_RABBITMQ_USERNAME"] = []byte(r.Username) } - if r.QueueName != "" { - data["ETOS_RABBITMQ_QUEUE_NAME"] = []byte(r.QueueName) - } - if r.QueueParams != "" { - data["ETOS_RABBITMQ_QUEUE_PARAMS"] = []byte(r.QueueParams) - } return data, nil } diff --git a/internal/extras/rabbitmq.go b/internal/extras/rabbitmq.go index 4b94c480..abd262f9 100644 --- a/internal/extras/rabbitmq.go +++ b/internal/extras/rabbitmq.go @@ -219,12 +219,6 @@ func (r *RabbitMQDeployment) secretData(name types.NamespacedName) (map[string][ if r.Username != "" { data["RABBITMQ_USERNAME"] = []byte(r.Username) } - if r.QueueName != "" { - data["RABBITMQ_QUEUE"] = []byte(r.QueueName) - } - if r.QueueParams != "" { - data["RABBITMQ_QUEUE_PARAMS"] = []byte(r.QueueParams) - } return data, nil } From 6a643366bfb3ed99e82e818e53ff5a458980aa5d Mon Sep 17 00:00:00 2001 From: Tobias Persson Date: Thu, 10 Oct 2024 14:26:14 +0200 Subject: [PATCH 02/11] Remove commented code --- api/v1alpha1/common_types.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/api/v1alpha1/common_types.go b/api/v1alpha1/common_types.go index dccd9693..10600e7f 100644 --- a/api/v1alpha1/common_types.go +++ b/api/v1alpha1/common_types.go @@ -128,6 +128,4 @@ type RabbitMQ struct { // +kubebuilder:default=/ // +optional Vhost string `json:"vhost"` - // QueueName string `json:"queueName,omitempty"` - // QueueParams string `json:"queueParams,omitempty"` } From ef092c05fb620e4ef8bfb441b96c477f808230bb Mon Sep 17 00:00:00 2001 From: Tobias Persson Date: Fri, 11 Oct 2024 10:13:36 +0200 Subject: [PATCH 03/11] Add routing key to the suite starter configmap --- internal/etos/suitestarter/suitestarter.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/etos/suitestarter/suitestarter.go b/internal/etos/suitestarter/suitestarter.go index 5958fe93..9cd89fd8 100644 --- a/internal/etos/suitestarter/suitestarter.go +++ b/internal/etos/suitestarter/suitestarter.go @@ -305,6 +305,7 @@ func (r *ETOSSuiteStarterDeployment) reconcileRolebinding(ctx context.Context, n // configmap creates a configmap resource definition for the ETOS suite runner. func (r *ETOSSuiteStarterDeployment) configmap(name types.NamespacedName, secretName string, cluster *etosv1alpha1.Cluster) *corev1.ConfigMap { + routingKey := fmt.Sprintf("eiffel.*.EiffelTestExecutionRecipeCollectionCreatedEvent.%s.*", cluster.Spec.ETOS.Config.RoutingKeyTag) data := map[string]string{ "SUITE_RUNNER": cluster.Spec.ETOS.SuiteRunner.Image.Image, "LOG_LISTENER": cluster.Spec.ETOS.SuiteRunner.LogListener.Image.Image, @@ -312,6 +313,7 @@ func (r *ETOSSuiteStarterDeployment) configmap(name types.NamespacedName, secret "ETOS_RABBITMQ_SECRET": secretName, "ETOS_ESR_TTL": r.Config.TTL, "ETOS_TERMINATION_GRACE_PERIOD": r.Config.GracePeriod, + "RABBITMQ_ROUTING_KEY": routingKey, } if r.Config.ObservabilityConfigmapName != "" { data["ETOS_OBSERVABILITY_CONFIGMAP"] = r.Config.ObservabilityConfigmapName From c7ae26ae7a8ac6c4c37abcae07e87981b8e86256 Mon Sep 17 00:00:00 2001 From: Tobias Persson Date: Fri, 18 Oct 2024 09:37:44 +0200 Subject: [PATCH 04/11] Clean up config management and add routing key to testrun suite runners --- api/v1alpha1/cluster_types.go | 2 +- api/v1alpha1/environment_types.go | 10 +- api/v1alpha1/environmentrequest_types.go | 2 +- api/v1alpha1/testrun_types.go | 4 +- ...s.eiffel-community.github.io_clusters.yaml | 5 +- ...mmunity.github.io_environmentrequests.yaml | 2 +- ...ffel-community.github.io_environments.yaml | 10 +- ...s.eiffel-community.github.io_testruns.yaml | 4 +- .../environmentrequest_controller.go | 23 +-- internal/controller/testrun_controller.go | 52 +---- internal/etos/api/api.go | 120 +++++++---- internal/etos/api/logarea.go | 2 +- internal/etos/etos.go | 154 ++++++++++---- internal/etos/suitestarter/suitestarter.go | 189 +++++++++--------- internal/extras/eventrepository.go | 102 +++++++--- 15 files changed, 383 insertions(+), 298 deletions(-) diff --git a/api/v1alpha1/cluster_types.go b/api/v1alpha1/cluster_types.go index 6c1e81d9..4107b45b 100644 --- a/api/v1alpha1/cluster_types.go +++ b/api/v1alpha1/cluster_types.go @@ -143,7 +143,7 @@ type ETOSSuiteStarter struct { // Provide a custom suite runner template. // +kubebuilder:default="" // +optional - SuiteRunnerTemplateConfigmapName string `json:"suiteRunnerTemplateConfigmapName,omitempty"` + SuiteRunnerTemplateSecretName string `json:"suiteRunnerTemplateSecretName,omitempty"` // +kubebuilder:default={"ttl": "3600", "gracePeriod": "300"} // +optional Config ETOSSuiteStarterConfig `json:"config"` diff --git a/api/v1alpha1/environment_types.go b/api/v1alpha1/environment_types.go index f7c69867..0ae2a324 100644 --- a/api/v1alpha1/environment_types.go +++ b/api/v1alpha1/environment_types.go @@ -27,15 +27,15 @@ type EnvironmentSpec struct { // Snake casing as to be compatible with ETR. - // +kubebuilder:validation:Pattern="[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}" + // +kubebuilder:validation:Pattern="^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$" SuiteID string `json:"suite_id"` - // +kubebuilder:validation:Pattern="[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}" + // +kubebuilder:validation:Pattern="^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$" SubSuiteID string `json:"sub_suite_id"` - // +kubebuilder:validation:Pattern="[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}" + // +kubebuilder:validation:Pattern="^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$" MainSuiteID string `json:"test_suite_started_id"` - // +kubebuilder:validation:Pattern="[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}" + // +kubebuilder:validation:Pattern="^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$" Artifact string `json:"artifact"` - // +kubebuilder:validation:Pattern="[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}" + // +kubebuilder:validation:Pattern="^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$" Context string `json:"context"` Priority int `json:"priority"` diff --git a/api/v1alpha1/environmentrequest_types.go b/api/v1alpha1/environmentrequest_types.go index 7aa9091d..8fd23c7c 100644 --- a/api/v1alpha1/environmentrequest_types.go +++ b/api/v1alpha1/environmentrequest_types.go @@ -48,7 +48,7 @@ type Splitter struct { // EnvironmentRequestSpec defines the desired state of EnvironmentRequest type EnvironmentRequestSpec struct { // ID is the ID for the environments generated. Will be generated if nil - // +kubebuilder:validation:Pattern="[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}" + // +kubebuilder:validation:Pattern="^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$" ID string `json:"id,omitempty"` Name string `json:"name,omitempty"` diff --git a/api/v1alpha1/testrun_types.go b/api/v1alpha1/testrun_types.go index d04781e0..6c1b0688 100644 --- a/api/v1alpha1/testrun_types.go +++ b/api/v1alpha1/testrun_types.go @@ -104,11 +104,11 @@ type TestRunSpec struct { Cluster string `json:"cluster,omitempty"` // ID is the test suite ID for this execution. Will be generated if nil - // +kubebuilder:validation:Pattern="[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}" + // +kubebuilder:validation:Pattern="^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$" ID string `json:"id,omitempty"` // Artifact is the ID of the software under test. - // +kubebuilder:validation:Pattern="[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}" + // +kubebuilder:validation:Pattern="^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$" Artifact string `json:"artifact"` // +optional diff --git a/config/crd/bases/etos.eiffel-community.github.io_clusters.yaml b/config/crd/bases/etos.eiffel-community.github.io_clusters.yaml index 763dc0be..5c871e80 100644 --- a/config/crd/bases/etos.eiffel-community.github.io_clusters.yaml +++ b/config/crd/bases/etos.eiffel-community.github.io_clusters.yaml @@ -86,6 +86,9 @@ spec: pull a container image type: string iutProviderSecret: + description: |- + The provider secrets are necessary in order to run ETOS the old way and not using the controller. + They can be removed from here when the suite starter is no longer in use. type: string logAreaProviderSecret: type: string @@ -346,7 +349,7 @@ spec: description: PullPolicy describes a policy for if/when to pull a container image type: string - suiteRunnerTemplateConfigmapName: + suiteRunnerTemplateSecretName: default: "" description: Provide a custom suite runner template. type: string diff --git a/config/crd/bases/etos.eiffel-community.github.io_environmentrequests.yaml b/config/crd/bases/etos.eiffel-community.github.io_environmentrequests.yaml index 7f09f4e4..90cb1b19 100644 --- a/config/crd/bases/etos.eiffel-community.github.io_environmentrequests.yaml +++ b/config/crd/bases/etos.eiffel-community.github.io_environmentrequests.yaml @@ -58,7 +58,7 @@ spec: id: description: ID is the ID for the environments generated. Will be generated if nil - pattern: '[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}' + pattern: ^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$ type: string identifier: type: string diff --git a/config/crd/bases/etos.eiffel-community.github.io_environments.yaml b/config/crd/bases/etos.eiffel-community.github.io_environments.yaml index e3ca901c..e9b9a409 100644 --- a/config/crd/bases/etos.eiffel-community.github.io_environments.yaml +++ b/config/crd/bases/etos.eiffel-community.github.io_environments.yaml @@ -47,10 +47,10 @@ spec: description: EnvironmentSpec defines the desired state of Environment properties: artifact: - pattern: '[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}' + pattern: ^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$ type: string context: - pattern: '[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}' + pattern: ^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$ type: string executor: x-kubernetes-preserve-unknown-fields: true @@ -122,15 +122,15 @@ spec: type: object type: array sub_suite_id: - pattern: '[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}' + pattern: ^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$ type: string suite_id: - pattern: '[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}' + pattern: ^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$ type: string test_runner: type: string test_suite_started_id: - pattern: '[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}' + pattern: ^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$ type: string required: - artifact diff --git a/config/crd/bases/etos.eiffel-community.github.io_testruns.yaml b/config/crd/bases/etos.eiffel-community.github.io_testruns.yaml index 967e87f7..d35f7211 100644 --- a/config/crd/bases/etos.eiffel-community.github.io_testruns.yaml +++ b/config/crd/bases/etos.eiffel-community.github.io_testruns.yaml @@ -63,7 +63,7 @@ spec: properties: artifact: description: Artifact is the ID of the software under test. - pattern: '[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}' + pattern: ^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$ type: string cluster: description: Name of the ETOS cluster to execute the testrun in. @@ -83,7 +83,7 @@ spec: id: description: ID is the test suite ID for this execution. Will be generated if nil - pattern: '[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}' + pattern: ^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$ type: string identity: type: string diff --git a/internal/controller/environmentrequest_controller.go b/internal/controller/environmentrequest_controller.go index b40de743..5cff0b00 100644 --- a/internal/controller/environmentrequest_controller.go +++ b/internal/controller/environmentrequest_controller.go @@ -254,31 +254,10 @@ func (r EnvironmentRequestReconciler) environmentProviderJob(environmentrequest }, }, EnvFrom: []corev1.EnvFromSource{ - { - ConfigMapRef: &corev1.ConfigMapEnvSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: cluster, - }, - }, - }, - { - SecretRef: &corev1.SecretEnvSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: cluster, - }, - }, - }, - { - SecretRef: &corev1.SecretEnvSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: fmt.Sprintf("%s-rabbitmq", cluster), - }, - }, - }, { SecretRef: &corev1.SecretEnvSource{ LocalObjectReference: corev1.LocalObjectReference{ - Name: fmt.Sprintf("%s-messagebus", cluster), + Name: fmt.Sprintf("%s-environment-provider-cfg", cluster), }, }, }, diff --git a/internal/controller/testrun_controller.go b/internal/controller/testrun_controller.go index cf918b01..0530c085 100644 --- a/internal/controller/testrun_controller.go +++ b/internal/controller/testrun_controller.go @@ -523,17 +523,10 @@ func (r TestRunReconciler) suiteRunnerJob(tercc []byte, testrun *etosv1alpha1.Te ImagePullPolicy: testrun.Spec.LogListener.ImagePullPolicy, Command: []string{"python", "-u", "-m", "create_queue"}, EnvFrom: []corev1.EnvFromSource{ - { - ConfigMapRef: &corev1.ConfigMapEnvSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: testrun.Spec.Cluster, - }, - }, - }, { SecretRef: &corev1.SecretEnvSource{ LocalObjectReference: corev1.LocalObjectReference{ - Name: fmt.Sprintf("%s-messagebus", testrun.Spec.Cluster), + Name: fmt.Sprintf("%s-etos-suite-runner-cfg", testrun.Spec.Cluster), }, }, }, @@ -564,31 +557,10 @@ func (r TestRunReconciler) suiteRunnerJob(tercc []byte, testrun *etosv1alpha1.Te }, }, EnvFrom: []corev1.EnvFromSource{ - { - ConfigMapRef: &corev1.ConfigMapEnvSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: testrun.Spec.Cluster, - }, - }, - }, { SecretRef: &corev1.SecretEnvSource{ LocalObjectReference: corev1.LocalObjectReference{ - Name: testrun.Spec.Cluster, - }, - }, - }, - { - SecretRef: &corev1.SecretEnvSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: fmt.Sprintf("%s-rabbitmq", testrun.Spec.Cluster), - }, - }, - }, - { - SecretRef: &corev1.SecretEnvSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: fmt.Sprintf("%s-messagebus", testrun.Spec.Cluster), + Name: fmt.Sprintf("%s-etos-suite-runner-cfg", testrun.Spec.Cluster), }, }, }, @@ -606,6 +578,10 @@ func (r TestRunReconciler) suiteRunnerJob(tercc []byte, testrun *etosv1alpha1.Te Name: "IDENTITY", Value: testrun.Spec.Identity, }, + { + Name: "ETOS_ROUTING_KEY_TAG", + Value: fmt.Sprintf("%s-%s", testrun.Namespace, testrun.Spec.Cluster), + }, { Name: "TESTRUN", Value: testrun.Name, @@ -651,24 +627,10 @@ func (r TestRunReconciler) suiteRunnerJob(tercc []byte, testrun *etosv1alpha1.Te }, }, EnvFrom: []corev1.EnvFromSource{ - { - ConfigMapRef: &corev1.ConfigMapEnvSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: testrun.Spec.Cluster, - }, - }, - }, - { - SecretRef: &corev1.SecretEnvSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: testrun.Spec.Cluster, - }, - }, - }, { SecretRef: &corev1.SecretEnvSource{ LocalObjectReference: corev1.LocalObjectReference{ - Name: fmt.Sprintf("%s-messagebus", testrun.Spec.Cluster), + Name: fmt.Sprintf("%s-etos-suite-runner-cfg", testrun.Spec.Cluster), }, }, }, diff --git a/internal/etos/api/api.go b/internal/etos/api/api.go index 0c5c56eb..4e1d49ef 100644 --- a/internal/etos/api/api.go +++ b/internal/etos/api/api.go @@ -19,6 +19,7 @@ package api import ( "context" "fmt" + "maps" etosv1alpha1 "github.com/eiffel-community/etos/api/v1alpha1" appsv1 "k8s.io/api/apps/v1" @@ -47,12 +48,12 @@ type ETOSApiDeployment struct { Scheme *runtime.Scheme rabbitmqSecret string messagebusSecret string - configmap string + configSecret string } // NewETOSApiDeployment will create a new ETOS API reconciler. -func NewETOSApiDeployment(spec etosv1alpha1.ETOSAPI, scheme *runtime.Scheme, client client.Client, rabbitmqSecret string, messagebusSecret string, configmap string) *ETOSApiDeployment { - return &ETOSApiDeployment{spec, client, scheme, rabbitmqSecret, messagebusSecret, configmap} +func NewETOSApiDeployment(spec etosv1alpha1.ETOSAPI, scheme *runtime.Scheme, client client.Client, rabbitmqSecret string, messagebusSecret string, config string) *ETOSApiDeployment { + return &ETOSApiDeployment{spec, client, scheme, rabbitmqSecret, messagebusSecret, config} } // Reconcile will reconcile the ETOS API to its expected state. @@ -61,7 +62,11 @@ func (r *ETOSApiDeployment) Reconcile(ctx context.Context, cluster *etosv1alpha1 name := fmt.Sprintf("%s-etos-api", cluster.Name) namespacedName := types.NamespacedName{Name: name, Namespace: cluster.Namespace} - _, err = r.reconcileDeployment(ctx, namespacedName, cluster) + cfg, err := r.reconcileConfig(ctx, namespacedName, cluster) + if err != nil { + return err + } + _, err = r.reconcileDeployment(ctx, namespacedName, cfg.ObjectMeta.Name, cluster) if err != nil { return err } @@ -88,9 +93,33 @@ func (r *ETOSApiDeployment) Reconcile(ctx context.Context, cluster *etosv1alpha1 return nil } +// reconcileConfig will reconcile the secret to use as configuration for the ETOS API. +func (r *ETOSApiDeployment) reconcileConfig(ctx context.Context, name types.NamespacedName, owner metav1.Object) (*corev1.Secret, error) { + name = types.NamespacedName{Name: fmt.Sprintf("%s-cfg", name.Name), Namespace: name.Namespace} + target, err := r.config(ctx, name) + if err != nil { + return nil, err + } + if err := ctrl.SetControllerReference(owner, target, r.Scheme); err != nil { + return target, err + } + + secret := &corev1.Secret{} + if err := r.Get(ctx, name, secret); err != nil { + if !apierrors.IsNotFound(err) { + return secret, err + } + if err := r.Create(ctx, target); err != nil { + return target, err + } + return target, nil + } + return target, r.Patch(ctx, target, client.StrategicMergeFrom(secret)) +} + // reconcileDeployment will reconcile the ETOS API deployment to its expected state. -func (r *ETOSApiDeployment) reconcileDeployment(ctx context.Context, name types.NamespacedName, owner metav1.Object) (*appsv1.Deployment, error) { - target := r.deployment(name) +func (r *ETOSApiDeployment) reconcileDeployment(ctx context.Context, name types.NamespacedName, secretName string, owner metav1.Object) (*appsv1.Deployment, error) { + target := r.deployment(name, secretName) if err := ctrl.SetControllerReference(owner, target, r.Scheme); err != nil { return target, err } @@ -114,14 +143,15 @@ func (r *ETOSApiDeployment) reconcileDeployment(ctx context.Context, name types. // reconcileSecret will reconcile the ETOS API service account secret to its expected state. func (r *ETOSApiDeployment) reconcileSecret(ctx context.Context, name types.NamespacedName, owner metav1.Object) (*corev1.Secret, error) { - target := r.secret(name) + tokenName := types.NamespacedName{Name: fmt.Sprintf("%s-token", name.Name), Namespace: name.Namespace} + target := r.secret(tokenName, name) if err := ctrl.SetControllerReference(owner, target, r.Scheme); err != nil { return target, err } scheme.Scheme.Default(target) secret := &corev1.Secret{} - if err := r.Get(ctx, name, secret); err != nil { + if err := r.Get(ctx, tokenName, secret); err != nil { if !apierrors.IsNotFound(err) { return secret, err } @@ -219,11 +249,34 @@ func (r *ETOSApiDeployment) reconcileService(ctx context.Context, name types.Nam return target, r.Patch(ctx, target, client.StrategicMergeFrom(service)) } +// config creates a new Secret to be used as configuration for the ETOS API. +func (r *ETOSApiDeployment) config(ctx context.Context, name types.NamespacedName) (*corev1.Secret, error) { + eiffel := &corev1.Secret{} + if err := r.Get(ctx, types.NamespacedName{Name: r.rabbitmqSecret, Namespace: name.Namespace}, eiffel); err != nil { + return nil, err + } + etos := &corev1.Secret{} + if err := r.Get(ctx, types.NamespacedName{Name: r.messagebusSecret, Namespace: name.Namespace}, etos); err != nil { + return nil, err + } + config := &corev1.Secret{} + if err := r.Get(ctx, types.NamespacedName{Name: r.configSecret, Namespace: name.Namespace}, config); err != nil { + return nil, err + } + data := map[string][]byte{} + maps.Copy(data, eiffel.Data) + maps.Copy(data, etos.Data) + maps.Copy(data, config.Data) + return &corev1.Secret{ + ObjectMeta: r.meta(name), + Data: data, + }, nil +} + // secret creates a secret resource definition for the ETOS API. -func (r *ETOSApiDeployment) secret(name types.NamespacedName) *corev1.Secret { +func (r *ETOSApiDeployment) secret(name, serviceAccountName types.NamespacedName) *corev1.Secret { meta := r.meta(name) - meta.Annotations["kubernetes.io/service-account.name"] = name.Name - name.Name = fmt.Sprintf("%s-token", name.Name) + meta.Annotations["kubernetes.io/service-account.name"] = serviceAccountName.Name return &corev1.Secret{ ObjectMeta: meta, Type: corev1.SecretTypeServiceAccountToken, @@ -306,7 +359,7 @@ func (r *ETOSApiDeployment) rolebinding(name types.NamespacedName) *rbacv1.RoleB } // deployment creates a deployment resource definition for the ETOS API. -func (r *ETOSApiDeployment) deployment(name types.NamespacedName) *appsv1.Deployment { +func (r *ETOSApiDeployment) deployment(name types.NamespacedName, secretName string) *appsv1.Deployment { return &appsv1.Deployment{ ObjectMeta: r.meta(name), Spec: appsv1.DeploymentSpec{ @@ -321,7 +374,7 @@ func (r *ETOSApiDeployment) deployment(name types.NamespacedName) *appsv1.Deploy ObjectMeta: r.meta(name), Spec: corev1.PodSpec{ ServiceAccountName: name.Name, - Containers: []corev1.Container{r.container(name)}, + Containers: []corev1.Container{r.container(name, secretName)}, Volumes: r.volumes(), }, }, @@ -345,7 +398,7 @@ func (r *ETOSApiDeployment) service(name types.NamespacedName) *corev1.Service { } // container creates the container resource for the ETOS API deployment. -func (r *ETOSApiDeployment) container(name types.NamespacedName) corev1.Container { +func (r *ETOSApiDeployment) container(name types.NamespacedName, secretName string) corev1.Container { probe := &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ @@ -383,8 +436,16 @@ func (r *ETOSApiDeployment) container(name types.NamespacedName) corev1.Containe VolumeMounts: r.volumeMounts(), LivenessProbe: probe, ReadinessProbe: probe, - EnvFrom: r.environmentFrom(), - Env: r.environment(), + EnvFrom: []corev1.EnvFromSource{ + { + SecretRef: &corev1.SecretEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: secretName, + }, + }, + }, + }, + Env: r.environment(), } } @@ -439,33 +500,6 @@ func (r *ETOSApiDeployment) volumes() []corev1.Volume { return vol } -// environmentFrom creates the environment resource for the ETOS API deployment. -func (r *ETOSApiDeployment) environmentFrom() []corev1.EnvFromSource { - return []corev1.EnvFromSource{ - { - SecretRef: &corev1.SecretEnvSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: r.rabbitmqSecret, - }, - }, - }, - { - SecretRef: &corev1.SecretEnvSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: r.messagebusSecret, - }, - }, - }, - { - ConfigMapRef: &corev1.ConfigMapEnvSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: r.configmap, - }, - }, - }, - } -} - // environment creates environment variables for providers if supplied. func (r *ETOSApiDeployment) environment() []corev1.EnvVar { env := []corev1.EnvVar{} diff --git a/internal/etos/api/logarea.go b/internal/etos/api/logarea.go index bd209a77..b02c48bc 100644 --- a/internal/etos/api/logarea.go +++ b/internal/etos/api/logarea.go @@ -222,7 +222,7 @@ func (r *ETOSLogAreaDeployment) container(name types.NamespacedName, cluster *et { SecretRef: &corev1.SecretEnvSource{ LocalObjectReference: corev1.LocalObjectReference{ - Name: cluster.Name, + Name: fmt.Sprintf("%s-encryption-key", cluster.Name), }, }, }, diff --git a/internal/etos/etos.go b/internal/etos/etos.go index e0e39461..43d4be15 100644 --- a/internal/etos/etos.go +++ b/internal/etos/etos.go @@ -19,6 +19,7 @@ package etos import ( "context" "fmt" + "maps" etosv1alpha1 "github.com/eiffel-community/etos/api/v1alpha1" etosapi "github.com/eiffel-community/etos/internal/etos/api" @@ -34,6 +35,7 @@ import ( "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" ) type ETOSDeployment struct { @@ -52,53 +54,70 @@ func NewETOSDeployment(spec etosv1alpha1.ETOS, scheme *runtime.Scheme, client cl // Reconcile will reconcile ETOS to its expected state. func (r *ETOSDeployment) Reconcile(ctx context.Context, cluster *etosv1alpha1.Cluster) error { var err error + logger := log.FromContext(ctx) namespacedName := types.NamespacedName{Name: cluster.Name, Namespace: cluster.Namespace} if _, err := r.reconcileIngress(ctx, namespacedName, cluster); err != nil { + logger.Error(err, "Ingress reconciliation failed") return err } _, err = r.reconcileRole(ctx, namespacedName, cluster) if err != nil { + logger.Error(err, "Role reconciliation failed") return err } _, err = r.reconcileServiceAccount(ctx, namespacedName, cluster) if err != nil { + logger.Error(err, "ServiceAccount reconciliation failed") return err } _, err = r.reconcileRolebinding(ctx, namespacedName, cluster) if err != nil { + logger.Error(err, "Rolebinding reconciliation failed") return err } - configmap, err := r.reconcileConfigmap(ctx, namespacedName, cluster) + config, err := r.reconcileConfig(ctx, namespacedName, cluster) if err != nil { + logger.Error(err, "Config reconciliation failed") return err } encryption, err := r.reconcileSecret(ctx, namespacedName, cluster) if err != nil { + logger.Error(err, "Secret reconciliation failed") return err } - api := etosapi.NewETOSApiDeployment(r.API, r.Scheme, r.Client, r.rabbitmqSecret, r.messagebusSecret, configmap.Name) + _, err = r.reconcileEnvironmentProviderConfig(ctx, namespacedName, encryption.ObjectMeta.Name, config.ObjectMeta.Name, cluster) + if err != nil { + logger.Error(err, "Environment provider config reconciliation failed") + return err + } + + api := etosapi.NewETOSApiDeployment(r.API, r.Scheme, r.Client, r.rabbitmqSecret, r.messagebusSecret, config.ObjectMeta.Name) if err := api.Reconcile(ctx, cluster); err != nil { + logger.Error(err, "ETOS API reconciliation failed") return err } sse := etosapi.NewETOSSSEDeployment(r.SSE, r.Scheme, r.Client) if err := sse.Reconcile(ctx, cluster); err != nil { + logger.Error(err, "ETOS SSE reconciliation failed") return err } logarea := etosapi.NewETOSLogAreaDeployment(r.LogArea, r.Scheme, r.Client) if err := logarea.Reconcile(ctx, cluster); err != nil { + logger.Error(err, "ETOS LogArea reconciliation failed") return err } - suitestarter := etossuitestarter.NewETOSSuiteStarterDeployment(r.SuiteStarter, r.Scheme, r.Client, r.rabbitmqSecret, r.messagebusSecret, configmap, encryption) + suitestarter := etossuitestarter.NewETOSSuiteStarterDeployment(r.SuiteStarter, r.Scheme, r.Client, r.rabbitmqSecret, r.messagebusSecret, config, encryption) if err := suitestarter.Reconcile(ctx, cluster); err != nil { + logger.Error(err, "ETOS SuiteStarter reconciliation failed") return err } @@ -202,29 +221,31 @@ func (r *ETOSDeployment) reconcileRolebinding(ctx context.Context, name types.Na return target, r.Patch(ctx, target, client.StrategicMergeFrom(rolebinding)) } -// reconcileConfigmap will reconcile the ETOS configmap to its expected state. -func (r *ETOSDeployment) reconcileConfigmap(ctx context.Context, name types.NamespacedName, cluster *etosv1alpha1.Cluster) (*corev1.ConfigMap, error) { - target := r.configmap(name, cluster) +// reconcileConfig will reconcile the ETOS config to its expected state. +func (r *ETOSDeployment) reconcileConfig(ctx context.Context, name types.NamespacedName, cluster *etosv1alpha1.Cluster) (*corev1.Secret, error) { + name = types.NamespacedName{Name: fmt.Sprintf("%s-cfg", name.Name), Namespace: name.Namespace} + target := r.config(name, cluster) if err := ctrl.SetControllerReference(cluster, target, r.Scheme); err != nil { return target, err } scheme.Scheme.Default(target) - configmap := &corev1.ConfigMap{} - if err := r.Get(ctx, name, configmap); err != nil { + secret := &corev1.Secret{} + if err := r.Get(ctx, name, secret); err != nil { if !apierrors.IsNotFound(err) { - return configmap, err + return secret, err } if err := r.Create(ctx, target); err != nil { return target, err } return target, nil } - return target, r.Patch(ctx, target, client.StrategicMergeFrom(configmap)) + return target, r.Patch(ctx, target, client.StrategicMergeFrom(secret)) } // reconcileSecret will reconcile the secret to its expected state. func (r *ETOSDeployment) reconcileSecret(ctx context.Context, name types.NamespacedName, owner metav1.Object) (*corev1.Secret, error) { + name = types.NamespacedName{Name: fmt.Sprintf("%s-encryption-key", name.Name), Namespace: name.Namespace} target, err := r.secret(ctx, name) if err != nil { return target, err @@ -250,6 +271,59 @@ func (r *ETOSDeployment) reconcileSecret(ctx context.Context, name types.Namespa return target, r.Patch(ctx, target, client.StrategicMergeFrom(secret)) } +// reconcileEnvironmentProviderConfig will reconcile the secret to use as configuration for the ETOS environment provider. +func (r *ETOSDeployment) reconcileEnvironmentProviderConfig(ctx context.Context, name types.NamespacedName, encryptionKeyName, configmapName string, owner metav1.Object) (*corev1.Secret, error) { + name = types.NamespacedName{Name: fmt.Sprintf("%s-environment-provider-cfg", name.Name), Namespace: name.Namespace} + target, err := r.environmentProviderConfig(ctx, name, encryptionKeyName, configmapName) + if err != nil { + return nil, err + } + if err := ctrl.SetControllerReference(owner, target, r.Scheme); err != nil { + return target, err + } + + secret := &corev1.Secret{} + if err := r.Get(ctx, name, secret); err != nil { + if !apierrors.IsNotFound(err) { + return secret, err + } + if err := r.Create(ctx, target); err != nil { + return target, err + } + return target, nil + } + return target, r.Patch(ctx, target, client.StrategicMergeFrom(secret)) +} + +// config creates a new Secret to be used as configuration for the ETOS API. +func (r *ETOSDeployment) environmentProviderConfig(ctx context.Context, name types.NamespacedName, encryptionKeyName, configmapName string) (*corev1.Secret, error) { + eiffel := &corev1.Secret{} + if err := r.Get(ctx, types.NamespacedName{Name: r.rabbitmqSecret, Namespace: name.Namespace}, eiffel); err != nil { + return nil, err + } + etos := &corev1.Secret{} + if err := r.Get(ctx, types.NamespacedName{Name: r.messagebusSecret, Namespace: name.Namespace}, etos); err != nil { + return nil, err + } + encryption := &corev1.Secret{} + if err := r.Get(ctx, types.NamespacedName{Name: encryptionKeyName, Namespace: name.Namespace}, encryption); err != nil { + return nil, err + } + config := &corev1.Secret{} + if err := r.Get(ctx, types.NamespacedName{Name: configmapName, Namespace: name.Namespace}, config); err != nil { + return nil, err + } + data := map[string][]byte{} + maps.Copy(data, eiffel.Data) + maps.Copy(data, etos.Data) + maps.Copy(data, encryption.Data) + maps.Copy(data, config.Data) + return &corev1.Secret{ + ObjectMeta: r.meta(name), + Data: data, + }, nil +} + // ingress creates an ingress resource definition for ETOS. func (r *ETOSDeployment) ingress(name types.NamespacedName) *networkingv1.Ingress { ingress := &networkingv1.Ingress{ @@ -264,8 +338,8 @@ func (r *ETOSDeployment) ingress(name types.NamespacedName) *networkingv1.Ingres return ingress } -// configmap creates a configmap definition for ETOS. -func (r *ETOSDeployment) configmap(name types.NamespacedName, cluster *etosv1alpha1.Cluster) *corev1.ConfigMap { +// config creates a secret definition for ETOS. +func (r *ETOSDeployment) config(name types.NamespacedName, cluster *etosv1alpha1.Cluster) *corev1.Secret { etosHost := name.Name if r.Ingress.Host != "" { etosHost = r.Ingress.Host @@ -279,45 +353,45 @@ func (r *ETOSDeployment) configmap(name types.NamespacedName, cluster *etosv1alp eventRepository = r.Config.ETOSEventRepositoryURL } - data := map[string]string{ - "ETOS_GRAPHQL_SERVER": eventRepository, - "ETOS_CLUSTER": cluster.Name, - "ETOS_NAMESPACE": cluster.Namespace, - "ENVIRONMENT_PROVIDER_SERVICE_ACCOUNT": fmt.Sprintf("%s-provider", cluster.Name), - "SOURCE_HOST": r.Config.Source, - "ETOS_API": etosApi, - "SUITE_RUNNER_IMAGE": cluster.Spec.ETOS.SuiteRunner.Image.Image, - "SUITE_RUNNER_IMAGE_PULL_POLICY": string(cluster.Spec.ETOS.SuiteRunner.ImagePullPolicy), - "LOG_LISTENER_IMAGE": cluster.Spec.ETOS.SuiteRunner.LogListener.Image.Image, - "LOG_LISTENER_IMAGE_PULL_POLICY": string(cluster.Spec.ETOS.SuiteRunner.LogListener.ImagePullPolicy), - "ENVIRONMENT_PROVIDER_IMAGE": cluster.Spec.ETOS.EnvironmentProvider.Image.Image, - "ENVIRONMENT_PROVIDER_IMAGE_PULL_POLICY": string(cluster.Spec.ETOS.EnvironmentProvider.ImagePullPolicy), - "ETR_VERSION": cluster.Spec.ETOS.TestRunner.Version, - "ETOS_ROUTING_KEY_TAG": cluster.Spec.ETOS.Config.RoutingKeyTag, + data := map[string][]byte{ + "ETOS_GRAPHQL_SERVER": []byte(eventRepository), + "ETOS_CLUSTER": []byte(cluster.Name), + "ETOS_NAMESPACE": []byte(cluster.Namespace), + "ENVIRONMENT_PROVIDER_SERVICE_ACCOUNT": []byte(fmt.Sprintf("%s-provider", cluster.Name)), + "SOURCE_HOST": []byte(r.Config.Source), + "ETOS_API": []byte(etosApi), + "SUITE_RUNNER_IMAGE": []byte(cluster.Spec.ETOS.SuiteRunner.Image.Image), + "SUITE_RUNNER_IMAGE_PULL_POLICY": []byte(string(cluster.Spec.ETOS.SuiteRunner.ImagePullPolicy)), + "LOG_LISTENER_IMAGE": []byte(cluster.Spec.ETOS.SuiteRunner.LogListener.Image.Image), + "LOG_LISTENER_IMAGE_PULL_POLICY": []byte(string(cluster.Spec.ETOS.SuiteRunner.LogListener.ImagePullPolicy)), + "ENVIRONMENT_PROVIDER_IMAGE": []byte(cluster.Spec.ETOS.EnvironmentProvider.Image.Image), + "ENVIRONMENT_PROVIDER_IMAGE_PULL_POLICY": []byte(string(cluster.Spec.ETOS.EnvironmentProvider.ImagePullPolicy)), + "ETR_VERSION": []byte(cluster.Spec.ETOS.TestRunner.Version), + "ETOS_ROUTING_KEY_TAG": []byte(cluster.Spec.ETOS.Config.RoutingKeyTag), - "ETOS_ETCD_HOST": cluster.Spec.Database.Etcd.Host, - "ETOS_ETCD_PORT": cluster.Spec.Database.Etcd.Port, + "ETOS_ETCD_HOST": []byte(cluster.Spec.Database.Etcd.Host), + "ETOS_ETCD_PORT": []byte(cluster.Spec.Database.Etcd.Port), - "DEV": r.Config.Dev, + "DEV": []byte(r.Config.Dev), // TODO: A few of these seem redundant - "ESR_WAIT_FOR_ENVIRONMENT_TIMEOUT": r.Config.EnvironmentTimeout, - "ETOS_WAIT_FOR_IUT_TIMEOUT": r.Config.EnvironmentTimeout, - "ETOS_EVENT_DATA_TIMEOUT": r.Config.EventDataTimeout, - "ENVIRONMENT_PROVIDER_EVENT_DATA_TIMEOUT": r.Config.EventDataTimeout, - "ENVIRONMENT_PROVIDER_TEST_SUITE_TIMEOUT": r.Config.TestSuiteTimeout, - "ETOS_TEST_SUITE_TIMEOUT": r.Config.TestSuiteTimeout, + "ESR_WAIT_FOR_ENVIRONMENT_TIMEOUT": []byte(r.Config.EnvironmentTimeout), + "ETOS_WAIT_FOR_IUT_TIMEOUT": []byte(r.Config.EnvironmentTimeout), + "ETOS_EVENT_DATA_TIMEOUT": []byte(r.Config.EventDataTimeout), + "ENVIRONMENT_PROVIDER_EVENT_DATA_TIMEOUT": []byte(r.Config.EventDataTimeout), + "ENVIRONMENT_PROVIDER_TEST_SUITE_TIMEOUT": []byte(r.Config.TestSuiteTimeout), + "ETOS_TEST_SUITE_TIMEOUT": []byte(r.Config.TestSuiteTimeout), } if cluster.Spec.ETOS.Config.TestRunRetention.Failure != nil { - data["TESTRUN_FAILURE_RETENTION"] = cluster.Spec.ETOS.Config.TestRunRetention.Failure.Duration.String() + data["TESTRUN_FAILURE_RETENTION"] = []byte(cluster.Spec.ETOS.Config.TestRunRetention.Failure.Duration.String()) } if cluster.Spec.ETOS.Config.TestRunRetention.Success != nil { - data["TESTRUN_SUCCESS_RETENTION"] = cluster.Spec.ETOS.Config.TestRunRetention.Success.Duration.String() + data["TESTRUN_SUCCESS_RETENTION"] = []byte(cluster.Spec.ETOS.Config.TestRunRetention.Success.Duration.String()) } if r.Config.Timezone != "" { - data["TZ"] = r.Config.Timezone + data["TZ"] = []byte(r.Config.Timezone) } - return &corev1.ConfigMap{ + return &corev1.Secret{ ObjectMeta: r.meta(name), Data: data, } diff --git a/internal/etos/suitestarter/suitestarter.go b/internal/etos/suitestarter/suitestarter.go index 9cd89fd8..0620452c 100644 --- a/internal/etos/suitestarter/suitestarter.go +++ b/internal/etos/suitestarter/suitestarter.go @@ -42,13 +42,13 @@ type ETOSSuiteStarterDeployment struct { Scheme *runtime.Scheme rabbitmqSecret string messagebusSecret string - etosConfigmap *corev1.ConfigMap + etosConfig *corev1.Secret encryptionSecret *corev1.Secret } // NewETOSSuiteStarterDeployment will create a new ETOS SuiteStarter reconciler. -func NewETOSSuiteStarterDeployment(spec etosv1alpha1.ETOSSuiteStarter, scheme *runtime.Scheme, client client.Client, rabbitmqSecret, messagebusSecret string, configmap *corev1.ConfigMap, encryption *corev1.Secret) *ETOSSuiteStarterDeployment { - return &ETOSSuiteStarterDeployment{spec, client, scheme, rabbitmqSecret, messagebusSecret, configmap, encryption} +func NewETOSSuiteStarterDeployment(spec etosv1alpha1.ETOSSuiteStarter, scheme *runtime.Scheme, client client.Client, rabbitmqSecret, messagebusSecret string, config *corev1.Secret, encryption *corev1.Secret) *ETOSSuiteStarterDeployment { + return &ETOSSuiteStarterDeployment{spec, client, scheme, rabbitmqSecret, messagebusSecret, config, encryption} } // Reconcile will reconcile the ETOS suite starter to its expected state. @@ -61,26 +61,26 @@ func (r *ETOSSuiteStarterDeployment) Reconcile(ctx context.Context, cluster *eto if err != nil { return err } + // This secret is in use when running the TestRun controller. When the suite starter is removed, this MUST still be created. secret, err := r.reconcileSuiteRunnerSecret(ctx, namespacedName, cluster) if err != nil { return err } - configmap, err := r.reconcileConfigmap(ctx, secret.ObjectMeta.Name, namespacedName, cluster) + cfg, err := r.reconcileConfig(ctx, secret.ObjectMeta.Name, namespacedName, cluster) if err != nil { return err } - template, err := r.reconcileTemplate(ctx, namespacedName, cluster) if err != nil { return err } var suiteRunnerTemplateName string - if r.SuiteRunnerTemplateConfigmapName == "" { + if r.SuiteRunnerTemplateSecretName == "" { suiteRunnerTemplateName = template.ObjectMeta.Name } else { - suiteRunnerTemplateName = r.SuiteRunnerTemplateConfigmapName + suiteRunnerTemplateName = r.SuiteRunnerTemplateSecretName } - _, err = r.reconcileDeployment(ctx, configmap.ObjectMeta.Name, suiteRunnerTemplateName, namespacedName, cluster) + _, err = r.reconcileDeployment(ctx, cfg.ObjectMeta.Name, suiteRunnerTemplateName, namespacedName, cluster) if err != nil { return err } @@ -124,13 +124,13 @@ func (r *ETOSSuiteStarterDeployment) reconcileSuiteRunnerServiceAccount(ctx cont } // reconcileSuiteRunnerSecret will reconcile the ETOS suite runner secret to its expected state. -func (r *ETOSSuiteStarterDeployment) reconcileSuiteRunnerSecret(ctx context.Context, name types.NamespacedName, owner metav1.Object) (*corev1.Secret, error) { - name.Name = fmt.Sprintf("%s-suite-runner", name.Name) +func (r *ETOSSuiteStarterDeployment) reconcileSuiteRunnerSecret(ctx context.Context, name types.NamespacedName, cluster *etosv1alpha1.Cluster) (*corev1.Secret, error) { + name.Name = fmt.Sprintf("%s-etos-suite-runner-cfg", cluster.ObjectMeta.Name) target, err := r.mergedSecret(ctx, name) if err != nil { return nil, err } - if err := ctrl.SetControllerReference(owner, target, r.Scheme); err != nil { + if err := ctrl.SetControllerReference(cluster, target, r.Scheme); err != nil { return target, err } @@ -148,53 +148,57 @@ func (r *ETOSSuiteStarterDeployment) reconcileSuiteRunnerSecret(ctx context.Cont } // reconcileConfigmap will reconcile the ETOS suite starter config to its expected state. -func (r *ETOSSuiteStarterDeployment) reconcileConfigmap(ctx context.Context, secretName string, name types.NamespacedName, cluster *etosv1alpha1.Cluster) (*corev1.ConfigMap, error) { - target := r.configmap(name, secretName, cluster) +func (r *ETOSSuiteStarterDeployment) reconcileConfig(ctx context.Context, secretName string, name types.NamespacedName, cluster *etosv1alpha1.Cluster) (*corev1.Secret, error) { + name = types.NamespacedName{Name: fmt.Sprintf("%s-cfg", name.Name), Namespace: name.Namespace} + target, err := r.config(ctx, name, secretName, cluster) + if err != nil { + return nil, err + } if err := ctrl.SetControllerReference(cluster, target, r.Scheme); err != nil { return target, err } - configmap := &corev1.ConfigMap{} - if err := r.Get(ctx, name, configmap); err != nil { + secret := &corev1.Secret{} + if err := r.Get(ctx, name, secret); err != nil { if !apierrors.IsNotFound(err) { - return configmap, err + return secret, err } if err := r.Create(ctx, target); err != nil { return target, err } return target, nil } - return target, r.Patch(ctx, target, client.StrategicMergeFrom(configmap)) + return target, r.Patch(ctx, target, client.StrategicMergeFrom(secret)) } // reconcileTemplate will reconcile the ETOS SuiteRunner template to its expected state. -func (r *ETOSSuiteStarterDeployment) reconcileTemplate(ctx context.Context, name types.NamespacedName, owner metav1.Object) (*corev1.ConfigMap, error) { - name.Name = fmt.Sprintf("%s-template", name.Name) +func (r *ETOSSuiteStarterDeployment) reconcileTemplate(ctx context.Context, name types.NamespacedName, owner metav1.Object) (*corev1.Secret, error) { + name = types.NamespacedName{Name: fmt.Sprintf("%s-template", name.Name), Namespace: name.Namespace} target := r.suiteRunnerTemplate(name) if err := ctrl.SetControllerReference(owner, target, r.Scheme); err != nil { return target, err } scheme.Scheme.Default(target) - configmap := &corev1.ConfigMap{} - if err := r.Get(ctx, name, configmap); err != nil { + secret := &corev1.Secret{} + if err := r.Get(ctx, name, secret); err != nil { if !apierrors.IsNotFound(err) { - return configmap, err + return secret, err } if err := r.Create(ctx, target); err != nil { return target, err } return target, nil } - if equality.Semantic.DeepDerivative(target.Data, configmap.Data) { - return configmap, nil + if equality.Semantic.DeepDerivative(target.Data, secret.Data) { + return secret, nil } - return target, r.Patch(ctx, target, client.StrategicMergeFrom(configmap)) + return target, r.Patch(ctx, target, client.StrategicMergeFrom(secret)) } // reconcileDeployment will reconcile the ETOS SuiteStarter deployment to its expected state. -func (r *ETOSSuiteStarterDeployment) reconcileDeployment(ctx context.Context, configmap string, suiteRunnerTemplate string, name types.NamespacedName, owner metav1.Object) (*appsv1.Deployment, error) { - target := r.deployment(name, configmap, suiteRunnerTemplate) +func (r *ETOSSuiteStarterDeployment) reconcileDeployment(ctx context.Context, secretName string, suiteRunnerTemplate string, name types.NamespacedName, owner metav1.Object) (*appsv1.Deployment, error) { + target := r.deployment(name, secretName, suiteRunnerTemplate) if err := ctrl.SetControllerReference(owner, target, r.Scheme); err != nil { return target, err } @@ -218,14 +222,15 @@ func (r *ETOSSuiteStarterDeployment) reconcileDeployment(ctx context.Context, co // reconcileSecret will reconcile the ETOS SuiteStarter service account secret to its expected state. func (r *ETOSSuiteStarterDeployment) reconcileSecret(ctx context.Context, name types.NamespacedName, owner metav1.Object) (*corev1.Secret, error) { - target := r.secret(name) + tokenName := types.NamespacedName{Name: fmt.Sprintf("%s-token", name.Name), Namespace: name.Namespace} + target := r.secret(tokenName, name) if err := ctrl.SetControllerReference(owner, target, r.Scheme); err != nil { return target, err } scheme.Scheme.Default(target) secret := &corev1.Secret{} - if err := r.Get(ctx, name, secret); err != nil { + if err := r.Get(ctx, tokenName, secret); err != nil { if !apierrors.IsNotFound(err) { return secret, err } @@ -303,39 +308,52 @@ func (r *ETOSSuiteStarterDeployment) reconcileRolebinding(ctx context.Context, n return target, r.Patch(ctx, target, client.StrategicMergeFrom(rolebinding)) } -// configmap creates a configmap resource definition for the ETOS suite runner. -func (r *ETOSSuiteStarterDeployment) configmap(name types.NamespacedName, secretName string, cluster *etosv1alpha1.Cluster) *corev1.ConfigMap { +// config creates a secret resource definition for the ETOS suite runner. +func (r *ETOSSuiteStarterDeployment) config(ctx context.Context, name types.NamespacedName, secretName string, cluster *etosv1alpha1.Cluster) (*corev1.Secret, error) { routingKey := fmt.Sprintf("eiffel.*.EiffelTestExecutionRecipeCollectionCreatedEvent.%s.*", cluster.Spec.ETOS.Config.RoutingKeyTag) - data := map[string]string{ - "SUITE_RUNNER": cluster.Spec.ETOS.SuiteRunner.Image.Image, - "LOG_LISTENER": cluster.Spec.ETOS.SuiteRunner.LogListener.Image.Image, - "ETOS_CONFIGMAP": r.etosConfigmap.ObjectMeta.Name, - "ETOS_RABBITMQ_SECRET": secretName, - "ETOS_ESR_TTL": r.Config.TTL, - "ETOS_TERMINATION_GRACE_PERIOD": r.Config.GracePeriod, - "RABBITMQ_ROUTING_KEY": routingKey, + data := map[string][]byte{ + "SUITE_RUNNER": []byte(cluster.Spec.ETOS.SuiteRunner.Image.Image), + "LOG_LISTENER": []byte(cluster.Spec.ETOS.SuiteRunner.LogListener.Image.Image), + "ETOS_CONFIGMAP": []byte(r.etosConfig.ObjectMeta.Name), + "ETOS_RABBITMQ_SECRET": []byte(secretName), + "ETOS_ESR_TTL": []byte(r.Config.TTL), + "ETOS_TERMINATION_GRACE_PERIOD": []byte(r.Config.GracePeriod), + "RABBITMQ_ROUTING_KEY": []byte(routingKey), + "RABBITMQ_QUEUE": []byte(r.EiffelQueueName), } if r.Config.ObservabilityConfigmapName != "" { - data["ETOS_OBSERVABILITY_CONFIGMAP"] = r.Config.ObservabilityConfigmapName + data["ETOS_OBSERVABILITY_CONFIGMAP"] = []byte(r.Config.ObservabilityConfigmapName) } if r.Config.SidecarImage != "" { - data["ETOS_SIDECAR_IMAGE"] = r.Config.SidecarImage + data["ETOS_SIDECAR_IMAGE"] = []byte(r.Config.SidecarImage) } if r.Config.OTELCollectorHost != "" { - data["OTEL_COLLECTOR_HOST"] = r.Config.OTELCollectorHost + data["OTEL_COLLECTOR_HOST"] = []byte(r.Config.OTELCollectorHost) + } + if r.EiffelQueueParams != "" { + data["RABBITMQ_QUEUE_PARAMS"] = []byte(r.EiffelQueueParams) + } + eiffel := &corev1.Secret{} + if err := r.Get(ctx, types.NamespacedName{Name: r.rabbitmqSecret, Namespace: name.Namespace}, eiffel); err != nil { + return nil, err } - maps.Copy(data, r.etosConfigmap.Data) - return &corev1.ConfigMap{ + etos := &corev1.Secret{} + if err := r.Get(ctx, types.NamespacedName{Name: r.messagebusSecret, Namespace: name.Namespace}, etos); err != nil { + return nil, err + } + maps.Copy(data, eiffel.Data) + maps.Copy(data, etos.Data) + maps.Copy(data, r.etosConfig.Data) + return &corev1.Secret{ ObjectMeta: r.meta(name), Data: data, - } + }, nil } // secret creates a secret resource definition for the ETOS SuiteStarter. -func (r *ETOSSuiteStarterDeployment) secret(name types.NamespacedName) *corev1.Secret { +func (r *ETOSSuiteStarterDeployment) secret(name, serviceAccountName types.NamespacedName) *corev1.Secret { meta := r.meta(name) - meta.Annotations["kubernetes.io/service-account.name"] = name.Name - name.Name = fmt.Sprintf("%s-token", name.Name) + meta.Annotations["kubernetes.io/service-account.name"] = serviceAccountName.Name return &corev1.Secret{ ObjectMeta: meta, Type: corev1.SecretTypeServiceAccountToken, @@ -356,6 +374,7 @@ func (r *ETOSSuiteStarterDeployment) mergedSecret(ctx context.Context, name type data := map[string][]byte{} maps.Copy(data, eiffel.Data) maps.Copy(data, etos.Data) + maps.Copy(data, r.etosConfig.Data) maps.Copy(data, r.encryptionSecret.Data) return &corev1.Secret{ ObjectMeta: r.meta(name), @@ -421,7 +440,7 @@ func (r *ETOSSuiteStarterDeployment) rolebinding(name types.NamespacedName) *rba } // deployment creates a deployment resource definition for the ETOS SuiteStarter. -func (r *ETOSSuiteStarterDeployment) deployment(name types.NamespacedName, configmap string, suiteRunnerTemplate string) *appsv1.Deployment { +func (r *ETOSSuiteStarterDeployment) deployment(name types.NamespacedName, secretName string, suiteRunnerTemplate string) *appsv1.Deployment { return &appsv1.Deployment{ ObjectMeta: r.meta(name), Spec: appsv1.DeploymentSpec{ @@ -436,14 +455,12 @@ func (r *ETOSSuiteStarterDeployment) deployment(name types.NamespacedName, confi ObjectMeta: r.meta(name), Spec: corev1.PodSpec{ ServiceAccountName: name.Name, - Containers: []corev1.Container{r.container(name, configmap)}, + Containers: []corev1.Container{r.container(name, secretName)}, Volumes: []corev1.Volume{{ Name: "suite-runner-template", VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: suiteRunnerTemplate, - }, + Secret: &corev1.SecretVolumeSource{ + SecretName: suiteRunnerTemplate, }, }, }}, @@ -454,13 +471,7 @@ func (r *ETOSSuiteStarterDeployment) deployment(name types.NamespacedName, confi } // container creates the container resource for the ETOS SuiteStarter deployment. -func (r *ETOSSuiteStarterDeployment) container(name types.NamespacedName, configmap string) corev1.Container { - env := []corev1.EnvVar{ - {Name: "RABBITMQ_QUEUE", Value: r.EiffelQueueName}, - } - if r.EiffelQueueParams != "" { - env = append(env, corev1.EnvVar{Name: "RABBITMQ_QUEUE_PARAMS", Value: r.EiffelQueueParams}) - } +func (r *ETOSSuiteStarterDeployment) container(name types.NamespacedName, secretName string) corev1.Container { return corev1.Container{ Name: name.Name, Image: r.Image.Image, @@ -475,8 +486,15 @@ func (r *ETOSSuiteStarterDeployment) container(name types.NamespacedName, config corev1.ResourceCPU: resource.MustParse("100m"), }, }, - EnvFrom: r.environment(configmap), - Env: env, + EnvFrom: []corev1.EnvFromSource{ + { + SecretRef: &corev1.SecretEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: secretName, + }, + }, + }, + }, VolumeMounts: []corev1.VolumeMount{ { Name: "suite-runner-template", @@ -487,10 +505,10 @@ func (r *ETOSSuiteStarterDeployment) container(name types.NamespacedName, config } } -// suiteRunnerTemplate creates a configmap resource for the ETOS SuiteStarter. -func (r *ETOSSuiteStarterDeployment) suiteRunnerTemplate(name types.NamespacedName) *corev1.ConfigMap { - data := map[string]string{ - "suite_runner_template.yaml": ` +// suiteRunnerTemplate creates a secret resource for the ETOS SuiteStarter. +func (r *ETOSSuiteStarterDeployment) suiteRunnerTemplate(name types.NamespacedName) *corev1.Secret { + data := map[string][]byte{ + "suite_runner_template.yaml": []byte(` apiVersion: batch/v1 kind: Job metadata: @@ -522,7 +540,7 @@ func (r *ETOSSuiteStarterDeployment) suiteRunnerTemplate(name types.NamespacedNa image: registry.nordix.org/eiffel/etos-log-listener:4969c9b2 command: ["python", "-u", "-m", "create_queue"] envFrom: - - configMapRef: + - secretRef: name: {etos_configmap} - secretRef: name: {etos_rabbitmq_secret} @@ -537,7 +555,7 @@ func (r *ETOSSuiteStarterDeployment) suiteRunnerTemplate(name types.NamespacedNa command: ['/kubexit/kubexit'] args: ['python', '-u', '-m', 'etos_suite_runner'] envFrom: - - configMapRef: + - secretRef: name: {etos_configmap} - secretRef: name: {etos_rabbitmq_secret} @@ -565,7 +583,7 @@ func (r *ETOSSuiteStarterDeployment) suiteRunnerTemplate(name types.NamespacedNa command: ['/kubexit/kubexit'] args: ['python', '-u', '-m', 'log_listener'] envFrom: - - configMapRef: + - secretRef: name: {etos_configmap} - secretRef: name: {etos_rabbitmq_secret} @@ -587,41 +605,14 @@ func (r *ETOSSuiteStarterDeployment) suiteRunnerTemplate(name types.NamespacedNa mountPath: /kubexit restartPolicy: Never backoffLimit: 0 - `, + `), } - return &corev1.ConfigMap{ + return &corev1.Secret{ ObjectMeta: r.meta(name), Data: data, } } -// environment creates the environment resource for the ETOS SuiteStarter deployment. -func (r *ETOSSuiteStarterDeployment) environment(configmap string) []corev1.EnvFromSource { - return []corev1.EnvFromSource{ - { - SecretRef: &corev1.SecretEnvSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: r.rabbitmqSecret, - }, - }, - }, - { - SecretRef: &corev1.SecretEnvSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: r.messagebusSecret, - }, - }, - }, - { - ConfigMapRef: &corev1.ConfigMapEnvSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: configmap, - }, - }, - }, - } -} - // meta creates the common meta resource for the ETOS SuiteStarter deployment. func (r *ETOSSuiteStarterDeployment) meta(name types.NamespacedName) metav1.ObjectMeta { return metav1.ObjectMeta{ diff --git a/internal/extras/eventrepository.go b/internal/extras/eventrepository.go index ca5ed05c..9e848e02 100644 --- a/internal/extras/eventrepository.go +++ b/internal/extras/eventrepository.go @@ -18,6 +18,7 @@ package extras import ( "context" "fmt" + "maps" "net/url" etosv1alpha1 "github.com/eiffel-community/etos/api/v1alpha1" @@ -55,7 +56,12 @@ func (r *EventRepositoryDeployment) Reconcile(ctx context.Context, cluster *etos name := fmt.Sprintf("%s-graphql", cluster.Name) namespacedName := types.NamespacedName{Name: name, Namespace: cluster.Namespace} - _, err := r.reconcileDeployment(ctx, namespacedName, cluster) + cfg, err := r.reconcileConfig(ctx, namespacedName, cluster) + if err != nil { + return err + } + + _, err = r.reconcileDeployment(ctx, namespacedName, cfg.ObjectMeta.Name, cluster) if err != nil { return err } @@ -77,9 +83,32 @@ func (r *EventRepositoryDeployment) Reconcile(ctx context.Context, cluster *etos return nil } +// reconcileConfig will reconcile the secret to use as configuration for the event repository. +func (r *EventRepositoryDeployment) reconcileConfig(ctx context.Context, name types.NamespacedName, owner metav1.Object) (*corev1.Secret, error) { + target, err := r.config(ctx, name) + if err != nil { + return nil, err + } + if err := ctrl.SetControllerReference(owner, target, r.Scheme); err != nil { + return target, err + } + + secret := &corev1.Secret{} + if err := r.Get(ctx, name, secret); err != nil { + if !apierrors.IsNotFound(err) { + return secret, err + } + if err := r.Create(ctx, target); err != nil { + return target, err + } + return target, nil + } + return target, r.Patch(ctx, target, client.StrategicMergeFrom(secret)) +} + // reconcileDeployment will reconcile the event repository deployment to its expected state. -func (r *EventRepositoryDeployment) reconcileDeployment(ctx context.Context, name types.NamespacedName, owner metav1.Object) (*appsv1.Deployment, error) { - target := r.deployment(name) +func (r *EventRepositoryDeployment) reconcileDeployment(ctx context.Context, name types.NamespacedName, secretName string, owner metav1.Object) (*appsv1.Deployment, error) { + target := r.deployment(name, secretName) if err := ctrl.SetControllerReference(owner, target, r.Scheme); err != nil { return target, err } @@ -158,8 +187,29 @@ func (r *EventRepositoryDeployment) reconcileIngress(ctx context.Context, name t return target, r.Patch(ctx, target, client.StrategicMergeFrom(ingress)) } +// config creates a new Secret to be used as configuration for the event repository. +func (r *EventRepositoryDeployment) config(ctx context.Context, name types.NamespacedName) (*corev1.Secret, error) { + eiffel := &corev1.Secret{} + if err := r.Get(ctx, types.NamespacedName{Name: r.rabbitmqSecret, Namespace: name.Namespace}, eiffel); err != nil { + return nil, err + } + etos := &corev1.Secret{} + if err := r.Get(ctx, types.NamespacedName{Name: r.mongodbSecret, Namespace: name.Namespace}, etos); err != nil { + return nil, err + } + data := map[string][]byte{} + maps.Copy(data, eiffel.Data) + maps.Copy(data, etos.Data) + data["RABBITMQ_QUEUE"] = []byte(r.EventRepository.EiffelQueueName) + data["RABBITMQ_QUEUE_PARAMS"] = []byte(r.EventRepository.EiffelQueueParams) + return &corev1.Secret{ + ObjectMeta: r.meta(name), + Data: data, + }, nil +} + // deployment will create a deployment resource definition for the event repository. -func (r *EventRepositoryDeployment) deployment(name types.NamespacedName) *appsv1.Deployment { +func (r *EventRepositoryDeployment) deployment(name types.NamespacedName, secretName string) *appsv1.Deployment { return &appsv1.Deployment{ ObjectMeta: r.meta(name), Spec: appsv1.DeploymentSpec{ @@ -169,7 +219,7 @@ func (r *EventRepositoryDeployment) deployment(name types.NamespacedName) *appsv Template: corev1.PodTemplateSpec{ ObjectMeta: r.meta(name), Spec: corev1.PodSpec{ - Containers: r.containers(name), + Containers: r.containers(name, secretName), }, }, }, @@ -214,7 +264,7 @@ func (r *EventRepositoryDeployment) meta(name types.NamespacedName) metav1.Objec } // containers will create a container resource definition for the event repository deployment. -func (r *EventRepositoryDeployment) containers(name types.NamespacedName) []corev1.Container { +func (r *EventRepositoryDeployment) containers(name types.NamespacedName, secretName string) []corev1.Container { return []corev1.Container{ { Name: fmt.Sprintf("%s-api", name.Name), @@ -227,7 +277,15 @@ func (r *EventRepositoryDeployment) containers(name types.NamespacedName) []core Protocol: "TCP", }, }, - EnvFrom: r.environment(), + EnvFrom: []corev1.EnvFromSource{ + { + SecretRef: &corev1.SecretEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: secretName, + }, + }, + }, + }, }, { Name: fmt.Sprintf("%s-storage", name.Name), Image: r.Storage.Image, @@ -237,29 +295,13 @@ func (r *EventRepositoryDeployment) containers(name types.NamespacedName) []core "-m", "eiffel_graphql_api.storage", }, - EnvFrom: r.environment(), - Env: []corev1.EnvVar{ - {Name: "RABBITMQ_QUEUE", Value: r.EventRepository.EiffelQueueName}, - {Name: "RABBITMQ_QUEUE_PARAMS", Value: r.EventRepository.EiffelQueueParams}, - }, - }, - } -} - -// environment will create an environment resource definition for the event repository deployment. -func (r *EventRepositoryDeployment) environment() []corev1.EnvFromSource { - return []corev1.EnvFromSource{ - { - SecretRef: &corev1.SecretEnvSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: r.mongodbSecret, - }, - }, - }, - { - SecretRef: &corev1.SecretEnvSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: r.rabbitmqSecret, + EnvFrom: []corev1.EnvFromSource{ + { + SecretRef: &corev1.SecretEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: secretName, + }, + }, }, }, }, From 5140b12392aa5e1ce5afd62635178e013ca57d91 Mon Sep 17 00:00:00 2001 From: Tobias Persson Date: Fri, 18 Oct 2024 12:52:23 +0200 Subject: [PATCH 05/11] Add skaffold for local development --- .dockerignore | 4 +++- skaffold.yaml | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 skaffold.yaml diff --git a/.dockerignore b/.dockerignore index a3aab7af..0cbfff3d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,5 @@ # More info: https://docs.docker.com/engine/reference/builder/#dockerignore-file # Ignore build and test binaries. -bin/ +bin +dist/install.yaml +config diff --git a/skaffold.yaml b/skaffold.yaml new file mode 100644 index 00000000..28960919 --- /dev/null +++ b/skaffold.yaml @@ -0,0 +1,17 @@ +apiVersion: skaffold/v4beta11 +kind: Config +metadata: + name: etos +build: + artifacts: + - image: registry.nordix.org/eiffel/etos-controller + docker: + dockerfile: Dockerfile + hooks: + after: + - command: + - make + - build-installer +manifests: + rawYaml: + - dist/install.yaml From 43248de595ed4b826d4c24b8c255f6d71327c57d Mon Sep 17 00:00:00 2001 From: Tobias Persson Date: Mon, 21 Oct 2024 11:43:46 +0200 Subject: [PATCH 06/11] Logging improvements --- internal/etos/api/api.go | 45 ++++++++++++----- internal/etos/api/logarea.go | 21 +++++--- internal/etos/api/sse.go | 33 ++++++++---- internal/etos/etos.go | 39 ++++++++------ internal/etos/suitestarter/suitestarter.go | 58 ++++++++++++++------- internal/extras/eventrepository.go | 31 +++++++++--- internal/extras/messagebus.go | 59 ++++++++++++---------- internal/extras/mongodb.go | 26 +++++++--- internal/extras/rabbitmq.go | 59 ++++++++++++---------- 9 files changed, 239 insertions(+), 132 deletions(-) diff --git a/internal/etos/api/api.go b/internal/etos/api/api.go index 4e1d49ef..f5e4d84a 100644 --- a/internal/etos/api/api.go +++ b/internal/etos/api/api.go @@ -22,6 +22,7 @@ import ( "maps" etosv1alpha1 "github.com/eiffel-community/etos/api/v1alpha1" + "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -35,6 +36,7 @@ import ( "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" ) var ( @@ -60,41 +62,49 @@ func NewETOSApiDeployment(spec etosv1alpha1.ETOSAPI, scheme *runtime.Scheme, cli func (r *ETOSApiDeployment) Reconcile(ctx context.Context, cluster *etosv1alpha1.Cluster) error { var err error name := fmt.Sprintf("%s-etos-api", cluster.Name) + logger := log.FromContext(ctx, "Reconciler", "ETOSApi", "BaseName", name) namespacedName := types.NamespacedName{Name: name, Namespace: cluster.Namespace} - cfg, err := r.reconcileConfig(ctx, namespacedName, cluster) + cfg, err := r.reconcileConfig(ctx, logger, namespacedName, cluster) if err != nil { + logger.Error(err, "Failed to reconcile the config for the ETOS API") return err } - _, err = r.reconcileDeployment(ctx, namespacedName, cfg.ObjectMeta.Name, cluster) + _, err = r.reconcileDeployment(ctx, logger, namespacedName, cfg.ObjectMeta.Name, cluster) if err != nil { + logger.Error(err, "Failed to reconcile the deployment for the ETOS API") return err } - _, err = r.reconcileSecret(ctx, namespacedName, cluster) + _, err = r.reconcileSecret(ctx, logger, namespacedName, cluster) if err != nil { + logger.Error(err, "Failed to reconcile the secret for the ETOS API") return err } - _, err = r.reconcileRole(ctx, namespacedName, cluster) + _, err = r.reconcileRole(ctx, logger, namespacedName, cluster) if err != nil { + logger.Error(err, "Failed to reconcile the role for the ETOS API") return err } - _, err = r.reconcileServiceAccount(ctx, namespacedName, cluster) + _, err = r.reconcileServiceAccount(ctx, logger, namespacedName, cluster) if err != nil { + logger.Error(err, "Failed to reconcile the service account for the ETOS API") return err } - _, err = r.reconcileRolebinding(ctx, namespacedName, cluster) + _, err = r.reconcileRolebinding(ctx, logger, namespacedName, cluster) if err != nil { + logger.Error(err, "Failed to reconcile the role binding for the ETOS API") return err } - _, err = r.reconcileService(ctx, namespacedName, cluster) + _, err = r.reconcileService(ctx, logger, namespacedName, cluster) if err != nil { + logger.Error(err, "Failed to reconcile the service for the ETOS API") return err } return nil } // reconcileConfig will reconcile the secret to use as configuration for the ETOS API. -func (r *ETOSApiDeployment) reconcileConfig(ctx context.Context, name types.NamespacedName, owner metav1.Object) (*corev1.Secret, error) { +func (r *ETOSApiDeployment) reconcileConfig(ctx context.Context, logger logr.Logger, name types.NamespacedName, owner metav1.Object) (*corev1.Secret, error) { name = types.NamespacedName{Name: fmt.Sprintf("%s-cfg", name.Name), Namespace: name.Namespace} target, err := r.config(ctx, name) if err != nil { @@ -109,6 +119,7 @@ func (r *ETOSApiDeployment) reconcileConfig(ctx context.Context, name types.Name if !apierrors.IsNotFound(err) { return secret, err } + logger.Info("Creating a new config for the ETOS API") if err := r.Create(ctx, target); err != nil { return target, err } @@ -118,7 +129,7 @@ func (r *ETOSApiDeployment) reconcileConfig(ctx context.Context, name types.Name } // reconcileDeployment will reconcile the ETOS API deployment to its expected state. -func (r *ETOSApiDeployment) reconcileDeployment(ctx context.Context, name types.NamespacedName, secretName string, owner metav1.Object) (*appsv1.Deployment, error) { +func (r *ETOSApiDeployment) reconcileDeployment(ctx context.Context, logger logr.Logger, name types.NamespacedName, secretName string, owner metav1.Object) (*appsv1.Deployment, error) { target := r.deployment(name, secretName) if err := ctrl.SetControllerReference(owner, target, r.Scheme); err != nil { return target, err @@ -130,6 +141,7 @@ func (r *ETOSApiDeployment) reconcileDeployment(ctx context.Context, name types. if !apierrors.IsNotFound(err) { return deployment, err } + logger.Info("Creating a new deployment for the ETOS API") if err := r.Create(ctx, target); err != nil { return target, err } @@ -142,7 +154,7 @@ func (r *ETOSApiDeployment) reconcileDeployment(ctx context.Context, name types. } // reconcileSecret will reconcile the ETOS API service account secret to its expected state. -func (r *ETOSApiDeployment) reconcileSecret(ctx context.Context, name types.NamespacedName, owner metav1.Object) (*corev1.Secret, error) { +func (r *ETOSApiDeployment) reconcileSecret(ctx context.Context, logger logr.Logger, name types.NamespacedName, owner metav1.Object) (*corev1.Secret, error) { tokenName := types.NamespacedName{Name: fmt.Sprintf("%s-token", name.Name), Namespace: name.Namespace} target := r.secret(tokenName, name) if err := ctrl.SetControllerReference(owner, target, r.Scheme); err != nil { @@ -155,6 +167,7 @@ func (r *ETOSApiDeployment) reconcileSecret(ctx context.Context, name types.Name if !apierrors.IsNotFound(err) { return secret, err } + logger.Info("Creating a new secret for the ETOS API service account") if err := r.Create(ctx, target); err != nil { return target, err } @@ -167,7 +180,7 @@ func (r *ETOSApiDeployment) reconcileSecret(ctx context.Context, name types.Name } // reconcileRole will reconcile the ETOS API service account role to its expected state. -func (r *ETOSApiDeployment) reconcileRole(ctx context.Context, name types.NamespacedName, owner metav1.Object) (*rbacv1.Role, error) { +func (r *ETOSApiDeployment) reconcileRole(ctx context.Context, logger logr.Logger, name types.NamespacedName, owner metav1.Object) (*rbacv1.Role, error) { labelName := name.Name name.Name = fmt.Sprintf("%s:sa:esr-handler", name.Name) @@ -181,6 +194,7 @@ func (r *ETOSApiDeployment) reconcileRole(ctx context.Context, name types.Namesp if !apierrors.IsNotFound(err) { return role, err } + logger.Info("Creating a new role for the ETOS API") if err := r.Create(ctx, target); err != nil { return target, err } @@ -190,7 +204,7 @@ func (r *ETOSApiDeployment) reconcileRole(ctx context.Context, name types.Namesp } // reconcileServiceAccount will reconcile the ETOS API service account to its expected state. -func (r *ETOSApiDeployment) reconcileServiceAccount(ctx context.Context, name types.NamespacedName, owner metav1.Object) (*corev1.ServiceAccount, error) { +func (r *ETOSApiDeployment) reconcileServiceAccount(ctx context.Context, logger logr.Logger, name types.NamespacedName, owner metav1.Object) (*corev1.ServiceAccount, error) { target := r.serviceaccount(name) if err := ctrl.SetControllerReference(owner, target, r.Scheme); err != nil { return target, err @@ -201,6 +215,7 @@ func (r *ETOSApiDeployment) reconcileServiceAccount(ctx context.Context, name ty if !apierrors.IsNotFound(err) { return serviceaccount, err } + logger.Info("Creating a new service account for the ETOS API") if err := r.Create(ctx, target); err != nil { return target, err } @@ -210,7 +225,7 @@ func (r *ETOSApiDeployment) reconcileServiceAccount(ctx context.Context, name ty } // reconcileRolebinding will reconcile the ETOS API service account role binding to its expected state. -func (r *ETOSApiDeployment) reconcileRolebinding(ctx context.Context, name types.NamespacedName, owner metav1.Object) (*rbacv1.RoleBinding, error) { +func (r *ETOSApiDeployment) reconcileRolebinding(ctx context.Context, logger logr.Logger, name types.NamespacedName, owner metav1.Object) (*rbacv1.RoleBinding, error) { target := r.rolebinding(name) if err := ctrl.SetControllerReference(owner, target, r.Scheme); err != nil { return target, err @@ -221,6 +236,7 @@ func (r *ETOSApiDeployment) reconcileRolebinding(ctx context.Context, name types if !apierrors.IsNotFound(err) { return rolebinding, err } + logger.Info("Creating a rolebinding for the ETOS API") if err := r.Create(ctx, target); err != nil { return target, err } @@ -230,7 +246,7 @@ func (r *ETOSApiDeployment) reconcileRolebinding(ctx context.Context, name types } // reconcileService will reconcile the ETOS API service to its expected state. -func (r *ETOSApiDeployment) reconcileService(ctx context.Context, name types.NamespacedName, owner metav1.Object) (*corev1.Service, error) { +func (r *ETOSApiDeployment) reconcileService(ctx context.Context, logger logr.Logger, name types.NamespacedName, owner metav1.Object) (*corev1.Service, error) { target := r.service(name) if err := ctrl.SetControllerReference(owner, target, r.Scheme); err != nil { return target, err @@ -241,6 +257,7 @@ func (r *ETOSApiDeployment) reconcileService(ctx context.Context, name types.Nam if !apierrors.IsNotFound(err) { return service, err } + logger.Info("Creating a new kubernetes service for the ETOS API") if err := r.Create(ctx, target); err != nil { return target, err } diff --git a/internal/etos/api/logarea.go b/internal/etos/api/logarea.go index b02c48bc..b7997492 100644 --- a/internal/etos/api/logarea.go +++ b/internal/etos/api/logarea.go @@ -21,6 +21,7 @@ import ( "fmt" etosv1alpha1 "github.com/eiffel-community/etos/api/v1alpha1" + "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" @@ -33,6 +34,7 @@ import ( "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" ) var ( @@ -55,25 +57,29 @@ func NewETOSLogAreaDeployment(spec etosv1alpha1.ETOSLogArea, scheme *runtime.Sch func (r *ETOSLogAreaDeployment) Reconcile(ctx context.Context, cluster *etosv1alpha1.Cluster) error { var err error name := fmt.Sprintf("%s-etos-logarea", cluster.Name) + logger := log.FromContext(ctx, "Reconciler", "ETOSLogArea", "BaseName", name) namespacedName := types.NamespacedName{Name: name, Namespace: cluster.Namespace} - _, err = r.reconcileDeployment(ctx, namespacedName, cluster) + _, err = r.reconcileDeployment(ctx, logger, namespacedName, cluster) if err != nil { + logger.Error(err, "Failed to reconcile the deployment for the ETOS LogArea") return err } - _, err = r.reconcileServiceAccount(ctx, namespacedName, cluster) + _, err = r.reconcileServiceAccount(ctx, logger, namespacedName, cluster) if err != nil { + logger.Error(err, "Failed to reconcile the service account for the ETOS LogArea") return err } - _, err = r.reconcileService(ctx, namespacedName, cluster) + _, err = r.reconcileService(ctx, logger, namespacedName, cluster) if err != nil { + logger.Error(err, "Failed to reconcile the service for the ETOS LogArea") return err } return nil } // reconcileDeployment will reconcile the ETOS logarea deployment to its expected state. -func (r *ETOSLogAreaDeployment) reconcileDeployment(ctx context.Context, name types.NamespacedName, cluster *etosv1alpha1.Cluster) (*appsv1.Deployment, error) { +func (r *ETOSLogAreaDeployment) reconcileDeployment(ctx context.Context, logger logr.Logger, name types.NamespacedName, cluster *etosv1alpha1.Cluster) (*appsv1.Deployment, error) { target := r.deployment(name, cluster) if err := ctrl.SetControllerReference(cluster, target, r.Scheme); err != nil { return target, err @@ -85,6 +91,7 @@ func (r *ETOSLogAreaDeployment) reconcileDeployment(ctx context.Context, name ty if !apierrors.IsNotFound(err) { return deployment, err } + logger.Info("Creating a new deployment for the ETOS LogArea") if err := r.Create(ctx, target); err != nil { return target, err } @@ -97,7 +104,7 @@ func (r *ETOSLogAreaDeployment) reconcileDeployment(ctx context.Context, name ty } // reconcileServiceAccount will reconcile the ETOS logarea service account to its expected state. -func (r *ETOSLogAreaDeployment) reconcileServiceAccount(ctx context.Context, name types.NamespacedName, owner metav1.Object) (*corev1.ServiceAccount, error) { +func (r *ETOSLogAreaDeployment) reconcileServiceAccount(ctx context.Context, logger logr.Logger, name types.NamespacedName, owner metav1.Object) (*corev1.ServiceAccount, error) { target := r.serviceaccount(name) if err := ctrl.SetControllerReference(owner, target, r.Scheme); err != nil { return target, err @@ -107,6 +114,7 @@ func (r *ETOSLogAreaDeployment) reconcileServiceAccount(ctx context.Context, nam if !apierrors.IsNotFound(err) { return serviceaccount, err } + logger.Info("Creating a new service account for the ETOS LogArea") if err := r.Create(ctx, target); err != nil { return target, err } @@ -116,7 +124,7 @@ func (r *ETOSLogAreaDeployment) reconcileServiceAccount(ctx context.Context, nam } // reconcileService will reconcile the ETOS logarea service to its expected state. -func (r *ETOSLogAreaDeployment) reconcileService(ctx context.Context, name types.NamespacedName, owner metav1.Object) (*corev1.Service, error) { +func (r *ETOSLogAreaDeployment) reconcileService(ctx context.Context, logger logr.Logger, name types.NamespacedName, owner metav1.Object) (*corev1.Service, error) { target := r.service(name) if err := ctrl.SetControllerReference(owner, target, r.Scheme); err != nil { return target, err @@ -126,6 +134,7 @@ func (r *ETOSLogAreaDeployment) reconcileService(ctx context.Context, name types if !apierrors.IsNotFound(err) { return service, err } + logger.Info("Creating a new kubernetes service for the ETOS LogArea") if err := r.Create(ctx, target); err != nil { return target, err } diff --git a/internal/etos/api/sse.go b/internal/etos/api/sse.go index b47e6151..8ea5a842 100644 --- a/internal/etos/api/sse.go +++ b/internal/etos/api/sse.go @@ -21,6 +21,7 @@ import ( "fmt" etosv1alpha1 "github.com/eiffel-community/etos/api/v1alpha1" + "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -34,6 +35,7 @@ import ( "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" ) var ( @@ -56,34 +58,40 @@ func NewETOSSSEDeployment(spec etosv1alpha1.ETOSSSE, scheme *runtime.Scheme, cli func (r *ETOSSSEDeployment) Reconcile(ctx context.Context, cluster *etosv1alpha1.Cluster) error { var err error name := fmt.Sprintf("%s-etos-sse", cluster.Name) + logger := log.FromContext(ctx, "Reconciler", "ETOSSSE", "BaseName", name) namespacedName := types.NamespacedName{Name: name, Namespace: cluster.Namespace} - _, err = r.reconcileDeployment(ctx, namespacedName, cluster) + _, err = r.reconcileDeployment(ctx, logger, namespacedName, cluster) if err != nil { + logger.Error(err, "Failed to reconcile the deployment for the ETOS SSE") return err } - _, err = r.reconcileRole(ctx, namespacedName, cluster) + _, err = r.reconcileRole(ctx, logger, namespacedName, cluster) if err != nil { + logger.Error(err, "Failed to reconcile the role for the ETOS SSE") return err } - _, err = r.reconcileServiceAccount(ctx, namespacedName, cluster) + _, err = r.reconcileServiceAccount(ctx, logger, namespacedName, cluster) if err != nil { + logger.Error(err, "Failed to reconcile the service account for the ETOS SSE") return err } - _, err = r.reconcileRolebinding(ctx, namespacedName, cluster) + _, err = r.reconcileRolebinding(ctx, logger, namespacedName, cluster) if err != nil { + logger.Error(err, "Failed to reconcile the role binding for the ETOS SSE") return err } - _, err = r.reconcileService(ctx, namespacedName, cluster) + _, err = r.reconcileService(ctx, logger, namespacedName, cluster) if err != nil { + logger.Error(err, "Failed to reconcile the service for the ETOS SSE") return err } return nil } // reconcileDeployment will reconcile the ETOS SSE deployment to its expected state. -func (r *ETOSSSEDeployment) reconcileDeployment(ctx context.Context, name types.NamespacedName, owner metav1.Object) (*appsv1.Deployment, error) { +func (r *ETOSSSEDeployment) reconcileDeployment(ctx context.Context, logger logr.Logger, name types.NamespacedName, owner metav1.Object) (*appsv1.Deployment, error) { target := r.deployment(name) if err := ctrl.SetControllerReference(owner, target, r.Scheme); err != nil { return target, err @@ -95,6 +103,7 @@ func (r *ETOSSSEDeployment) reconcileDeployment(ctx context.Context, name types. if !apierrors.IsNotFound(err) { return deployment, err } + logger.Info("Creating a new deployment for the SSE server") if err := r.Create(ctx, target); err != nil { return target, err } @@ -107,7 +116,7 @@ func (r *ETOSSSEDeployment) reconcileDeployment(ctx context.Context, name types. } // reconcileRole will reconcile the ETOS SSE service account role to its expected state. -func (r *ETOSSSEDeployment) reconcileRole(ctx context.Context, name types.NamespacedName, owner metav1.Object) (*rbacv1.Role, error) { +func (r *ETOSSSEDeployment) reconcileRole(ctx context.Context, logger logr.Logger, name types.NamespacedName, owner metav1.Object) (*rbacv1.Role, error) { labelName := name.Name name.Name = fmt.Sprintf("%s:sa:esr-reader", name.Name) @@ -121,6 +130,7 @@ func (r *ETOSSSEDeployment) reconcileRole(ctx context.Context, name types.Namesp if !apierrors.IsNotFound(err) { return role, err } + logger.Info("Creating a new role for the SSE server") if err := r.Create(ctx, target); err != nil { return target, err } @@ -130,7 +140,7 @@ func (r *ETOSSSEDeployment) reconcileRole(ctx context.Context, name types.Namesp } // reconcileServiceAccount will reconcile the ETOS SSE service account to its expected state. -func (r *ETOSSSEDeployment) reconcileServiceAccount(ctx context.Context, name types.NamespacedName, owner metav1.Object) (*corev1.ServiceAccount, error) { +func (r *ETOSSSEDeployment) reconcileServiceAccount(ctx context.Context, logger logr.Logger, name types.NamespacedName, owner metav1.Object) (*corev1.ServiceAccount, error) { target := r.serviceaccount(name) if err := ctrl.SetControllerReference(owner, target, r.Scheme); err != nil { return target, err @@ -141,6 +151,7 @@ func (r *ETOSSSEDeployment) reconcileServiceAccount(ctx context.Context, name ty if !apierrors.IsNotFound(err) { return serviceaccount, err } + logger.Info("Creating a new service account for the SSE server") if err := r.Create(ctx, target); err != nil { return target, err } @@ -150,7 +161,7 @@ func (r *ETOSSSEDeployment) reconcileServiceAccount(ctx context.Context, name ty } // reconcileRolebinding will reconcile the ETOS SSE service account rolebinding to its expected state. -func (r *ETOSSSEDeployment) reconcileRolebinding(ctx context.Context, name types.NamespacedName, owner metav1.Object) (*rbacv1.RoleBinding, error) { +func (r *ETOSSSEDeployment) reconcileRolebinding(ctx context.Context, logger logr.Logger, name types.NamespacedName, owner metav1.Object) (*rbacv1.RoleBinding, error) { target := r.rolebinding(name) if err := ctrl.SetControllerReference(owner, target, r.Scheme); err != nil { return target, err @@ -161,6 +172,7 @@ func (r *ETOSSSEDeployment) reconcileRolebinding(ctx context.Context, name types if !apierrors.IsNotFound(err) { return rolebinding, err } + logger.Info("Creating a new role binding for the SSE server") if err := r.Create(ctx, target); err != nil { return target, err } @@ -170,7 +182,7 @@ func (r *ETOSSSEDeployment) reconcileRolebinding(ctx context.Context, name types } // reconcileService will reconcile the ETOS SSE service to its expected state. -func (r *ETOSSSEDeployment) reconcileService(ctx context.Context, name types.NamespacedName, owner metav1.Object) (*corev1.Service, error) { +func (r *ETOSSSEDeployment) reconcileService(ctx context.Context, logger logr.Logger, name types.NamespacedName, owner metav1.Object) (*corev1.Service, error) { target := r.service(name) if err := ctrl.SetControllerReference(owner, target, r.Scheme); err != nil { return target, err @@ -180,6 +192,7 @@ func (r *ETOSSSEDeployment) reconcileService(ctx context.Context, name types.Nam if !apierrors.IsNotFound(err) { return service, err } + logger.Info("Creating a new kubernetes service for the SSE server") if err := r.Create(ctx, target); err != nil { return target, err } diff --git a/internal/etos/etos.go b/internal/etos/etos.go index 43d4be15..8bda18f1 100644 --- a/internal/etos/etos.go +++ b/internal/etos/etos.go @@ -24,6 +24,7 @@ import ( etosv1alpha1 "github.com/eiffel-community/etos/api/v1alpha1" etosapi "github.com/eiffel-community/etos/internal/etos/api" etossuitestarter "github.com/eiffel-community/etos/internal/etos/suitestarter" + "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -54,44 +55,44 @@ func NewETOSDeployment(spec etosv1alpha1.ETOS, scheme *runtime.Scheme, client cl // Reconcile will reconcile ETOS to its expected state. func (r *ETOSDeployment) Reconcile(ctx context.Context, cluster *etosv1alpha1.Cluster) error { var err error - logger := log.FromContext(ctx) + logger := log.FromContext(ctx, "Reconciler", "ETOS", "BaseName", cluster.Name) namespacedName := types.NamespacedName{Name: cluster.Name, Namespace: cluster.Namespace} - if _, err := r.reconcileIngress(ctx, namespacedName, cluster); err != nil { + if _, err := r.reconcileIngress(ctx, logger, namespacedName, cluster); err != nil { logger.Error(err, "Ingress reconciliation failed") return err } - _, err = r.reconcileRole(ctx, namespacedName, cluster) + _, err = r.reconcileRole(ctx, logger, namespacedName, cluster) if err != nil { logger.Error(err, "Role reconciliation failed") return err } - _, err = r.reconcileServiceAccount(ctx, namespacedName, cluster) + _, err = r.reconcileServiceAccount(ctx, logger, namespacedName, cluster) if err != nil { logger.Error(err, "ServiceAccount reconciliation failed") return err } - _, err = r.reconcileRolebinding(ctx, namespacedName, cluster) + _, err = r.reconcileRolebinding(ctx, logger, namespacedName, cluster) if err != nil { logger.Error(err, "Rolebinding reconciliation failed") return err } - config, err := r.reconcileConfig(ctx, namespacedName, cluster) + config, err := r.reconcileConfig(ctx, logger, namespacedName, cluster) if err != nil { logger.Error(err, "Config reconciliation failed") return err } - encryption, err := r.reconcileSecret(ctx, namespacedName, cluster) + encryption, err := r.reconcileSecret(ctx, logger, namespacedName, cluster) if err != nil { logger.Error(err, "Secret reconciliation failed") return err } - _, err = r.reconcileEnvironmentProviderConfig(ctx, namespacedName, encryption.ObjectMeta.Name, config.ObjectMeta.Name, cluster) + _, err = r.reconcileEnvironmentProviderConfig(ctx, logger, namespacedName, encryption.ObjectMeta.Name, config.ObjectMeta.Name, cluster) if err != nil { logger.Error(err, "Environment provider config reconciliation failed") return err @@ -125,7 +126,7 @@ func (r *ETOSDeployment) Reconcile(ctx context.Context, cluster *etosv1alpha1.Cl } // reconcileIngress will reconcile the ETOS ingress to its expected state. -func (r *ETOSDeployment) reconcileIngress(ctx context.Context, name types.NamespacedName, owner metav1.Object) (*networkingv1.Ingress, error) { +func (r *ETOSDeployment) reconcileIngress(ctx context.Context, logger logr.Logger, name types.NamespacedName, owner metav1.Object) (*networkingv1.Ingress, error) { target := r.ingress(name) if err := ctrl.SetControllerReference(owner, target, r.Scheme); err != nil { return target, err @@ -138,12 +139,14 @@ func (r *ETOSDeployment) reconcileIngress(ctx context.Context, name types.Namesp return ingress, err } if r.Ingress.Enabled { + logger.Info("ETOS ingress enabled, creating") if err := r.Create(ctx, target); err != nil { return target, err } } return target, nil } else if !r.Ingress.Enabled { + logger.Info("ETOS ingress disabled, removing") return nil, r.Delete(ctx, ingress) } if equality.Semantic.DeepDerivative(target.Spec, ingress.Spec) { @@ -153,7 +156,7 @@ func (r *ETOSDeployment) reconcileIngress(ctx context.Context, name types.Namesp } // reconcileRole will reconcile the ETOS API service account role to its expected state. -func (r *ETOSDeployment) reconcileRole(ctx context.Context, name types.NamespacedName, owner metav1.Object) (*rbacv1.Role, error) { +func (r *ETOSDeployment) reconcileRole(ctx context.Context, logger logr.Logger, name types.NamespacedName, owner metav1.Object) (*rbacv1.Role, error) { name.Name = fmt.Sprintf("%s-provider", name.Name) labelName := name.Name @@ -169,6 +172,7 @@ func (r *ETOSDeployment) reconcileRole(ctx context.Context, name types.Namespace if !apierrors.IsNotFound(err) { return role, err } + logger.Info("Creating an ETOS environment provider role", "roleName", name.Name) if err := r.Create(ctx, target); err != nil { return target, err } @@ -178,7 +182,7 @@ func (r *ETOSDeployment) reconcileRole(ctx context.Context, name types.Namespace } // reconcileServiceAccount will reconcile the ETOS API service account to its expected state. -func (r *ETOSDeployment) reconcileServiceAccount(ctx context.Context, name types.NamespacedName, owner metav1.Object) (*corev1.ServiceAccount, error) { +func (r *ETOSDeployment) reconcileServiceAccount(ctx context.Context, logger logr.Logger, name types.NamespacedName, owner metav1.Object) (*corev1.ServiceAccount, error) { name.Name = fmt.Sprintf("%s-provider", name.Name) target := r.serviceaccount(name) @@ -191,6 +195,7 @@ func (r *ETOSDeployment) reconcileServiceAccount(ctx context.Context, name types if !apierrors.IsNotFound(err) { return serviceaccount, err } + logger.Info("Creating an ETOS service account", "serviceAccountName", name.Name) if err := r.Create(ctx, target); err != nil { return target, err } @@ -200,7 +205,7 @@ func (r *ETOSDeployment) reconcileServiceAccount(ctx context.Context, name types } // reconcileRolebinding will reconcile the ETOS API service account role binding to its expected state. -func (r *ETOSDeployment) reconcileRolebinding(ctx context.Context, name types.NamespacedName, owner metav1.Object) (*rbacv1.RoleBinding, error) { +func (r *ETOSDeployment) reconcileRolebinding(ctx context.Context, logger logr.Logger, name types.NamespacedName, owner metav1.Object) (*rbacv1.RoleBinding, error) { name.Name = fmt.Sprintf("%s-provider", name.Name) target := r.rolebinding(name) @@ -213,6 +218,7 @@ func (r *ETOSDeployment) reconcileRolebinding(ctx context.Context, name types.Na if !apierrors.IsNotFound(err) { return rolebinding, err } + logger.Info("Creating role binding for ETOS", "roleBindingName", name.Name) if err := r.Create(ctx, target); err != nil { return target, err } @@ -222,7 +228,7 @@ func (r *ETOSDeployment) reconcileRolebinding(ctx context.Context, name types.Na } // reconcileConfig will reconcile the ETOS config to its expected state. -func (r *ETOSDeployment) reconcileConfig(ctx context.Context, name types.NamespacedName, cluster *etosv1alpha1.Cluster) (*corev1.Secret, error) { +func (r *ETOSDeployment) reconcileConfig(ctx context.Context, logger logr.Logger, name types.NamespacedName, cluster *etosv1alpha1.Cluster) (*corev1.Secret, error) { name = types.NamespacedName{Name: fmt.Sprintf("%s-cfg", name.Name), Namespace: name.Namespace} target := r.config(name, cluster) if err := ctrl.SetControllerReference(cluster, target, r.Scheme); err != nil { @@ -235,6 +241,7 @@ func (r *ETOSDeployment) reconcileConfig(ctx context.Context, name types.Namespa if !apierrors.IsNotFound(err) { return secret, err } + logger.Info("Creating the ETOS configmap") if err := r.Create(ctx, target); err != nil { return target, err } @@ -244,7 +251,7 @@ func (r *ETOSDeployment) reconcileConfig(ctx context.Context, name types.Namespa } // reconcileSecret will reconcile the secret to its expected state. -func (r *ETOSDeployment) reconcileSecret(ctx context.Context, name types.NamespacedName, owner metav1.Object) (*corev1.Secret, error) { +func (r *ETOSDeployment) reconcileSecret(ctx context.Context, logger logr.Logger, name types.NamespacedName, owner metav1.Object) (*corev1.Secret, error) { name = types.NamespacedName{Name: fmt.Sprintf("%s-encryption-key", name.Name), Namespace: name.Namespace} target, err := r.secret(ctx, name) if err != nil { @@ -260,6 +267,7 @@ func (r *ETOSDeployment) reconcileSecret(ctx context.Context, name types.Namespa if !apierrors.IsNotFound(err) { return secret, err } + logger.Info("Creating the ETOS encryption key secret") if err := r.Create(ctx, target); err != nil { return target, err } @@ -272,7 +280,7 @@ func (r *ETOSDeployment) reconcileSecret(ctx context.Context, name types.Namespa } // reconcileEnvironmentProviderConfig will reconcile the secret to use as configuration for the ETOS environment provider. -func (r *ETOSDeployment) reconcileEnvironmentProviderConfig(ctx context.Context, name types.NamespacedName, encryptionKeyName, configmapName string, owner metav1.Object) (*corev1.Secret, error) { +func (r *ETOSDeployment) reconcileEnvironmentProviderConfig(ctx context.Context, logger logr.Logger, name types.NamespacedName, encryptionKeyName, configmapName string, owner metav1.Object) (*corev1.Secret, error) { name = types.NamespacedName{Name: fmt.Sprintf("%s-environment-provider-cfg", name.Name), Namespace: name.Namespace} target, err := r.environmentProviderConfig(ctx, name, encryptionKeyName, configmapName) if err != nil { @@ -287,6 +295,7 @@ func (r *ETOSDeployment) reconcileEnvironmentProviderConfig(ctx context.Context, if !apierrors.IsNotFound(err) { return secret, err } + logger.Info("Creating the ETOS environment provider configmap") if err := r.Create(ctx, target); err != nil { return target, err } diff --git a/internal/etos/suitestarter/suitestarter.go b/internal/etos/suitestarter/suitestarter.go index 0620452c..d6e98281 100644 --- a/internal/etos/suitestarter/suitestarter.go +++ b/internal/etos/suitestarter/suitestarter.go @@ -22,6 +22,7 @@ import ( "maps" etosv1alpha1 "github.com/eiffel-community/etos/api/v1alpha1" + "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -34,6 +35,7 @@ import ( "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" ) type ETOSSuiteStarterDeployment struct { @@ -55,23 +57,28 @@ func NewETOSSuiteStarterDeployment(spec etosv1alpha1.ETOSSuiteStarter, scheme *r func (r *ETOSSuiteStarterDeployment) Reconcile(ctx context.Context, cluster *etosv1alpha1.Cluster) error { var err error name := fmt.Sprintf("%s-etos-suite-starter", cluster.Name) + logger := log.FromContext(ctx, "Reconciler", "ETOSSuiteStarter", "BaseName", name) namespacedName := types.NamespacedName{Name: name, Namespace: cluster.Namespace} // There is an assumption in the suite starter that the service account is named 'etos-sa'. - _, err = r.reconcileSuiteRunnerServiceAccount(ctx, types.NamespacedName{Name: "etos-sa", Namespace: cluster.Namespace}, cluster) + _, err = r.reconcileSuiteRunnerServiceAccount(ctx, logger, types.NamespacedName{Name: "etos-sa", Namespace: cluster.Namespace}, cluster) if err != nil { + logger.Error(err, "Failed to reconcile the Suite runner service account") return err } // This secret is in use when running the TestRun controller. When the suite starter is removed, this MUST still be created. - secret, err := r.reconcileSuiteRunnerSecret(ctx, namespacedName, cluster) + secret, err := r.reconcileSuiteRunnerSecret(ctx, logger, namespacedName, cluster) if err != nil { + logger.Error(err, "Failed to reconcile the Suite runner secret") return err } - cfg, err := r.reconcileConfig(ctx, secret.ObjectMeta.Name, namespacedName, cluster) + cfg, err := r.reconcileConfig(ctx, logger, secret.ObjectMeta.Name, namespacedName, cluster) if err != nil { + logger.Error(err, "Failed to reconcile the Suite starter config") return err } - template, err := r.reconcileTemplate(ctx, namespacedName, cluster) + template, err := r.reconcileTemplate(ctx, logger, namespacedName, cluster) if err != nil { + logger.Error(err, "Failed to reconcile the Suite runner template") return err } var suiteRunnerTemplateName string @@ -80,31 +87,37 @@ func (r *ETOSSuiteStarterDeployment) Reconcile(ctx context.Context, cluster *eto } else { suiteRunnerTemplateName = r.SuiteRunnerTemplateSecretName } - _, err = r.reconcileDeployment(ctx, cfg.ObjectMeta.Name, suiteRunnerTemplateName, namespacedName, cluster) + logger.Info("Suite runner template", "suiteRunnerTemplateName", suiteRunnerTemplateName) + _, err = r.reconcileDeployment(ctx, logger, cfg.ObjectMeta.Name, suiteRunnerTemplateName, namespacedName, cluster) if err != nil { + logger.Error(err, "Failed to reconcile the deployment for the ETOS Suite Starter") return err } - _, err = r.reconcileSecret(ctx, namespacedName, cluster) + _, err = r.reconcileSecret(ctx, logger, namespacedName, cluster) if err != nil { + logger.Error(err, "Failed to reconcile the secret for the ETOS Suite Starter") return err } - _, err = r.reconcileRole(ctx, namespacedName, cluster) + _, err = r.reconcileRole(ctx, logger, namespacedName, cluster) if err != nil { + logger.Error(err, "Failed to reconcile the role for the ETOS Suite Starter") return err } - _, err = r.reconcileServiceAccount(ctx, namespacedName, cluster) + _, err = r.reconcileServiceAccount(ctx, logger, namespacedName, cluster) if err != nil { + logger.Error(err, "Failed to reconcile the service account for the ETOS Suite Starter") return err } - _, err = r.reconcileRolebinding(ctx, namespacedName, cluster) + _, err = r.reconcileRolebinding(ctx, logger, namespacedName, cluster) if err != nil { + logger.Error(err, "Failed to reconcile the role binding for the ETOS Suite Starter") return err } return err } // reconcileSuiteRunnerServiceAccount will reconcile the ETOS SuiteStarter service account to its expected state. -func (r *ETOSSuiteStarterDeployment) reconcileSuiteRunnerServiceAccount(ctx context.Context, name types.NamespacedName, owner metav1.Object) (*corev1.ServiceAccount, error) { +func (r *ETOSSuiteStarterDeployment) reconcileSuiteRunnerServiceAccount(ctx context.Context, logger logr.Logger, name types.NamespacedName, owner metav1.Object) (*corev1.ServiceAccount, error) { target := r.serviceaccount(name) if err := ctrl.SetControllerReference(owner, target, r.Scheme); err != nil { return target, err @@ -115,6 +128,7 @@ func (r *ETOSSuiteStarterDeployment) reconcileSuiteRunnerServiceAccount(ctx cont if !apierrors.IsNotFound(err) { return serviceaccount, err } + logger.Info("Creating a new service account for the suite runner") if err := r.Create(ctx, target); err != nil { return target, err } @@ -124,7 +138,7 @@ func (r *ETOSSuiteStarterDeployment) reconcileSuiteRunnerServiceAccount(ctx cont } // reconcileSuiteRunnerSecret will reconcile the ETOS suite runner secret to its expected state. -func (r *ETOSSuiteStarterDeployment) reconcileSuiteRunnerSecret(ctx context.Context, name types.NamespacedName, cluster *etosv1alpha1.Cluster) (*corev1.Secret, error) { +func (r *ETOSSuiteStarterDeployment) reconcileSuiteRunnerSecret(ctx context.Context, logger logr.Logger, name types.NamespacedName, cluster *etosv1alpha1.Cluster) (*corev1.Secret, error) { name.Name = fmt.Sprintf("%s-etos-suite-runner-cfg", cluster.ObjectMeta.Name) target, err := r.mergedSecret(ctx, name) if err != nil { @@ -139,6 +153,7 @@ func (r *ETOSSuiteStarterDeployment) reconcileSuiteRunnerSecret(ctx context.Cont if !apierrors.IsNotFound(err) { return secret, err } + logger.Info("Creating a new secret for the suite runner") if err := r.Create(ctx, target); err != nil { return target, err } @@ -148,7 +163,7 @@ func (r *ETOSSuiteStarterDeployment) reconcileSuiteRunnerSecret(ctx context.Cont } // reconcileConfigmap will reconcile the ETOS suite starter config to its expected state. -func (r *ETOSSuiteStarterDeployment) reconcileConfig(ctx context.Context, secretName string, name types.NamespacedName, cluster *etosv1alpha1.Cluster) (*corev1.Secret, error) { +func (r *ETOSSuiteStarterDeployment) reconcileConfig(ctx context.Context, logger logr.Logger, secretName string, name types.NamespacedName, cluster *etosv1alpha1.Cluster) (*corev1.Secret, error) { name = types.NamespacedName{Name: fmt.Sprintf("%s-cfg", name.Name), Namespace: name.Namespace} target, err := r.config(ctx, name, secretName, cluster) if err != nil { @@ -163,6 +178,7 @@ func (r *ETOSSuiteStarterDeployment) reconcileConfig(ctx context.Context, secret if !apierrors.IsNotFound(err) { return secret, err } + logger.Info("Creating a new config for the suite starter") if err := r.Create(ctx, target); err != nil { return target, err } @@ -172,7 +188,7 @@ func (r *ETOSSuiteStarterDeployment) reconcileConfig(ctx context.Context, secret } // reconcileTemplate will reconcile the ETOS SuiteRunner template to its expected state. -func (r *ETOSSuiteStarterDeployment) reconcileTemplate(ctx context.Context, name types.NamespacedName, owner metav1.Object) (*corev1.Secret, error) { +func (r *ETOSSuiteStarterDeployment) reconcileTemplate(ctx context.Context, logger logr.Logger, name types.NamespacedName, owner metav1.Object) (*corev1.Secret, error) { name = types.NamespacedName{Name: fmt.Sprintf("%s-template", name.Name), Namespace: name.Namespace} target := r.suiteRunnerTemplate(name) if err := ctrl.SetControllerReference(owner, target, r.Scheme); err != nil { @@ -185,6 +201,7 @@ func (r *ETOSSuiteStarterDeployment) reconcileTemplate(ctx context.Context, name if !apierrors.IsNotFound(err) { return secret, err } + logger.Info("Creating a new suite runner template for the suite starter") if err := r.Create(ctx, target); err != nil { return target, err } @@ -197,7 +214,7 @@ func (r *ETOSSuiteStarterDeployment) reconcileTemplate(ctx context.Context, name } // reconcileDeployment will reconcile the ETOS SuiteStarter deployment to its expected state. -func (r *ETOSSuiteStarterDeployment) reconcileDeployment(ctx context.Context, secretName string, suiteRunnerTemplate string, name types.NamespacedName, owner metav1.Object) (*appsv1.Deployment, error) { +func (r *ETOSSuiteStarterDeployment) reconcileDeployment(ctx context.Context, logger logr.Logger, secretName string, suiteRunnerTemplate string, name types.NamespacedName, owner metav1.Object) (*appsv1.Deployment, error) { target := r.deployment(name, secretName, suiteRunnerTemplate) if err := ctrl.SetControllerReference(owner, target, r.Scheme); err != nil { return target, err @@ -209,6 +226,7 @@ func (r *ETOSSuiteStarterDeployment) reconcileDeployment(ctx context.Context, se if !apierrors.IsNotFound(err) { return deployment, err } + logger.Info("Creating a new deployment for the suite starter") if err := r.Create(ctx, target); err != nil { return target, err } @@ -221,7 +239,7 @@ func (r *ETOSSuiteStarterDeployment) reconcileDeployment(ctx context.Context, se } // reconcileSecret will reconcile the ETOS SuiteStarter service account secret to its expected state. -func (r *ETOSSuiteStarterDeployment) reconcileSecret(ctx context.Context, name types.NamespacedName, owner metav1.Object) (*corev1.Secret, error) { +func (r *ETOSSuiteStarterDeployment) reconcileSecret(ctx context.Context, logger logr.Logger, name types.NamespacedName, owner metav1.Object) (*corev1.Secret, error) { tokenName := types.NamespacedName{Name: fmt.Sprintf("%s-token", name.Name), Namespace: name.Namespace} target := r.secret(tokenName, name) if err := ctrl.SetControllerReference(owner, target, r.Scheme); err != nil { @@ -234,6 +252,7 @@ func (r *ETOSSuiteStarterDeployment) reconcileSecret(ctx context.Context, name t if !apierrors.IsNotFound(err) { return secret, err } + logger.Info("Creating a new secret for the suite starter service account") if err := r.Create(ctx, target); err != nil { return target, err } @@ -246,7 +265,7 @@ func (r *ETOSSuiteStarterDeployment) reconcileSecret(ctx context.Context, name t } // reconcileRole will reconcile the ETOS SuiteStarter service account role to its expected state. -func (r *ETOSSuiteStarterDeployment) reconcileRole(ctx context.Context, name types.NamespacedName, owner metav1.Object) (*rbacv1.Role, error) { +func (r *ETOSSuiteStarterDeployment) reconcileRole(ctx context.Context, logger logr.Logger, name types.NamespacedName, owner metav1.Object) (*rbacv1.Role, error) { labelName := name.Name name.Name = fmt.Sprintf("%s:sa:esr-handler", name.Name) @@ -260,6 +279,7 @@ func (r *ETOSSuiteStarterDeployment) reconcileRole(ctx context.Context, name typ if !apierrors.IsNotFound(err) { return role, err } + logger.Info("Creating a new role for the suite starter") if err := r.Create(ctx, target); err != nil { return target, err } @@ -269,7 +289,7 @@ func (r *ETOSSuiteStarterDeployment) reconcileRole(ctx context.Context, name typ } // reconcileServiceAccount will reconcile the ETOS SuiteStarter service account to its expected state. -func (r *ETOSSuiteStarterDeployment) reconcileServiceAccount(ctx context.Context, name types.NamespacedName, owner metav1.Object) (*corev1.ServiceAccount, error) { +func (r *ETOSSuiteStarterDeployment) reconcileServiceAccount(ctx context.Context, logger logr.Logger, name types.NamespacedName, owner metav1.Object) (*corev1.ServiceAccount, error) { target := r.serviceaccount(name) if err := ctrl.SetControllerReference(owner, target, r.Scheme); err != nil { return target, err @@ -280,6 +300,7 @@ func (r *ETOSSuiteStarterDeployment) reconcileServiceAccount(ctx context.Context if !apierrors.IsNotFound(err) { return serviceaccount, err } + logger.Info("Creating a new service account for the suite starter") if err := r.Create(ctx, target); err != nil { return target, err } @@ -289,7 +310,7 @@ func (r *ETOSSuiteStarterDeployment) reconcileServiceAccount(ctx context.Context } // reconcileRolebinding will reconcile the ETOS SuiteStarter service account role binding to its expected state. -func (r *ETOSSuiteStarterDeployment) reconcileRolebinding(ctx context.Context, name types.NamespacedName, owner metav1.Object) (*rbacv1.RoleBinding, error) { +func (r *ETOSSuiteStarterDeployment) reconcileRolebinding(ctx context.Context, logger logr.Logger, name types.NamespacedName, owner metav1.Object) (*rbacv1.RoleBinding, error) { target := r.rolebinding(name) if err := ctrl.SetControllerReference(owner, target, r.Scheme); err != nil { return target, err @@ -300,6 +321,7 @@ func (r *ETOSSuiteStarterDeployment) reconcileRolebinding(ctx context.Context, n if !apierrors.IsNotFound(err) { return rolebinding, err } + logger.Info("Creating a new role binding for the suite starter") if err := r.Create(ctx, target); err != nil { return target, err } diff --git a/internal/extras/eventrepository.go b/internal/extras/eventrepository.go index 9e848e02..0a351eae 100644 --- a/internal/extras/eventrepository.go +++ b/internal/extras/eventrepository.go @@ -22,6 +22,7 @@ import ( "net/url" etosv1alpha1 "github.com/eiffel-community/etos/api/v1alpha1" + "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" @@ -33,6 +34,7 @@ import ( "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" ) var graphqlPort int32 = 5000 @@ -54,23 +56,28 @@ func NewEventRepositoryDeployment(spec *etosv1alpha1.EventRepository, scheme *ru // Reconcile will reconcile the event repository to its expected state. func (r *EventRepositoryDeployment) Reconcile(ctx context.Context, cluster *etosv1alpha1.Cluster) error { name := fmt.Sprintf("%s-graphql", cluster.Name) + logger := log.FromContext(ctx, "Reconciler", "EventRepository", "BaseName", name) namespacedName := types.NamespacedName{Name: name, Namespace: cluster.Namespace} - cfg, err := r.reconcileConfig(ctx, namespacedName, cluster) + cfg, err := r.reconcileConfig(ctx, logger, namespacedName, cluster) if err != nil { + logger.Error(err, "Failed to reconcile the EventRepository configuration") return err } - _, err = r.reconcileDeployment(ctx, namespacedName, cfg.ObjectMeta.Name, cluster) + _, err = r.reconcileDeployment(ctx, logger, namespacedName, cfg.ObjectMeta.Name, cluster) if err != nil { + logger.Error(err, "Failed to reconcile the EventRepository deployment") return err } - _, err = r.reconcileService(ctx, namespacedName, cluster) + _, err = r.reconcileService(ctx, logger, namespacedName, cluster) if err != nil { + logger.Error(err, "Failed to reconcile the EventRepository service") return err } - _, err = r.reconcileIngress(ctx, namespacedName, cluster) + _, err = r.reconcileIngress(ctx, logger, namespacedName, cluster) if err != nil { + logger.Error(err, "Failed to reconcile the EventRepository ingress") return err } if r.Ingress.Enabled { @@ -79,12 +86,13 @@ func (r *EventRepositoryDeployment) Reconcile(ctx context.Context, cluster *etos host = r.Ingress.Host } r.Host = fmt.Sprintf("http://%s/graphql", host) + logger.Info("Host for the EventRepository", "host", r.Host) } return nil } // reconcileConfig will reconcile the secret to use as configuration for the event repository. -func (r *EventRepositoryDeployment) reconcileConfig(ctx context.Context, name types.NamespacedName, owner metav1.Object) (*corev1.Secret, error) { +func (r *EventRepositoryDeployment) reconcileConfig(ctx context.Context, logger logr.Logger, name types.NamespacedName, owner metav1.Object) (*corev1.Secret, error) { target, err := r.config(ctx, name) if err != nil { return nil, err @@ -98,6 +106,7 @@ func (r *EventRepositoryDeployment) reconcileConfig(ctx context.Context, name ty if !apierrors.IsNotFound(err) { return secret, err } + logger.Info("Creating the configuration for an EventRepository") if err := r.Create(ctx, target); err != nil { return target, err } @@ -107,7 +116,7 @@ func (r *EventRepositoryDeployment) reconcileConfig(ctx context.Context, name ty } // reconcileDeployment will reconcile the event repository deployment to its expected state. -func (r *EventRepositoryDeployment) reconcileDeployment(ctx context.Context, name types.NamespacedName, secretName string, owner metav1.Object) (*appsv1.Deployment, error) { +func (r *EventRepositoryDeployment) reconcileDeployment(ctx context.Context, logger logr.Logger, name types.NamespacedName, secretName string, owner metav1.Object) (*appsv1.Deployment, error) { target := r.deployment(name, secretName) if err := ctrl.SetControllerReference(owner, target, r.Scheme); err != nil { return target, err @@ -120,12 +129,14 @@ func (r *EventRepositoryDeployment) reconcileDeployment(ctx context.Context, nam return deployment, err } if r.Deploy { + logger.Info("Creating a new EventRepository deployment") if err := r.Create(ctx, target); err != nil { return target, err } } return target, nil } else if !r.Deploy { + logger.Info("Removing the deployment for EventRepository") return nil, r.Delete(ctx, deployment) } if equality.Semantic.DeepDerivative(target.Spec, deployment.Spec) { @@ -135,7 +146,7 @@ func (r *EventRepositoryDeployment) reconcileDeployment(ctx context.Context, nam } // reconcileService will reconcile the event repository service to its expected state. -func (r *EventRepositoryDeployment) reconcileService(ctx context.Context, name types.NamespacedName, owner metav1.Object) (*corev1.Service, error) { +func (r *EventRepositoryDeployment) reconcileService(ctx context.Context, logger logr.Logger, name types.NamespacedName, owner metav1.Object) (*corev1.Service, error) { target := r.service(name) if err := ctrl.SetControllerReference(owner, target, r.Scheme); err != nil { return target, err @@ -147,19 +158,21 @@ func (r *EventRepositoryDeployment) reconcileService(ctx context.Context, name t return service, err } if r.Deploy { + logger.Info("Creating a new EventRepository kubernetes service") if err := r.Create(ctx, target); err != nil { return target, err } } return target, nil } else if !r.Deploy { + logger.Info("Removing the kubernetes service for EventRepository") return nil, r.Delete(ctx, service) } return target, r.Patch(ctx, target, client.StrategicMergeFrom(service)) } // reconcileIngress will reconcile the event repository ingress to its expected state. -func (r *EventRepositoryDeployment) reconcileIngress(ctx context.Context, name types.NamespacedName, owner metav1.Object) (*networkingv1.Ingress, error) { +func (r *EventRepositoryDeployment) reconcileIngress(ctx context.Context, logger logr.Logger, name types.NamespacedName, owner metav1.Object) (*networkingv1.Ingress, error) { target := r.ingress(name) if err := ctrl.SetControllerReference(owner, target, r.Scheme); err != nil { return target, err @@ -172,12 +185,14 @@ func (r *EventRepositoryDeployment) reconcileIngress(ctx context.Context, name t return ingress, err } if r.Ingress.Enabled { + logger.Info("Ingress enabled, creating a new ingress for the EventRepository") if err := r.Create(ctx, target); err != nil { return target, err } } return target, nil } else if !r.Ingress.Enabled { + logger.Info("Ingress disabled, removing ingress for the EventRepository") return nil, r.Delete(ctx, ingress) } diff --git a/internal/extras/messagebus.go b/internal/extras/messagebus.go index 2773527a..e1aa9b9f 100644 --- a/internal/extras/messagebus.go +++ b/internal/extras/messagebus.go @@ -20,6 +20,7 @@ import ( "fmt" etosv1alpha1 "github.com/eiffel-community/etos/api/v1alpha1" + "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" @@ -37,21 +38,19 @@ import ( type MessageBusDeployment struct { etosv1alpha1.RabbitMQ client.Client - ctx context.Context Scheme *runtime.Scheme SecretName string } // NewMessageBusDeployment will create a new messagebus reconciler. func NewMessageBusDeployment(spec etosv1alpha1.RabbitMQ, scheme *runtime.Scheme, client client.Client) *MessageBusDeployment { - return &MessageBusDeployment{spec, client, nil, scheme, ""} + return &MessageBusDeployment{spec, client, scheme, ""} } // Reconcile will reconcile the messagebus to its expected state. func (r *MessageBusDeployment) Reconcile(ctx context.Context, cluster *etosv1alpha1.Cluster) error { - logger := log.FromContext(ctx) - r.ctx = ctx name := fmt.Sprintf("%s-messagebus", cluster.Name) + logger := log.FromContext(ctx, "Reconciler", "MessageBus", "BaseName", name) namespacedName := types.NamespacedName{Name: name, Namespace: cluster.Namespace} if r.Deploy { logger.Info("Patching host & port when deploying RabbitMQ", "host", name, "port", rabbitmqPort) @@ -59,26 +58,29 @@ func (r *MessageBusDeployment) Reconcile(ctx context.Context, cluster *etosv1alp r.Port = fmt.Sprintf("%d", rabbitmqPort) } - secret, err := r.reconcileSecret(namespacedName, cluster) + secret, err := r.reconcileSecret(ctx, logger, namespacedName, cluster) if err != nil { + logger.Error(err, "Failed to reconcile the MessageBus secret") return err } r.SecretName = secret.Name - _, err = r.reconcileStatefulset(namespacedName, cluster) + _, err = r.reconcileStatefulset(ctx, logger, namespacedName, cluster) if err != nil { + logger.Error(err, "Failed to reconcile the MessageBus statefulset") return err } - _, err = r.reconcileService(namespacedName, cluster) + _, err = r.reconcileService(ctx, logger, namespacedName, cluster) if err != nil { + logger.Error(err, "Failed to reconcile the MessageBus service") return err } return nil } // reconcileSecret will reconcile the messagebus secret to its expected state. -func (r *MessageBusDeployment) reconcileSecret(name types.NamespacedName, owner metav1.Object) (*corev1.Secret, error) { - target, err := r.secret(name) +func (r *MessageBusDeployment) reconcileSecret(ctx context.Context, logger logr.Logger, name types.NamespacedName, owner metav1.Object) (*corev1.Secret, error) { + target, err := r.secret(ctx, name) if err != nil { return target, err } @@ -88,11 +90,12 @@ func (r *MessageBusDeployment) reconcileSecret(name types.NamespacedName, owner scheme.Scheme.Default(target) secret := &corev1.Secret{} - if err := r.Get(r.ctx, name, secret); err != nil { + if err := r.Get(ctx, name, secret); err != nil { if !apierrors.IsNotFound(err) { return secret, err } - if err := r.Create(r.ctx, target); err != nil { + logger.Info("Secret not found. Creating") + if err := r.Create(ctx, target); err != nil { return target, err } return target, nil @@ -100,60 +103,64 @@ func (r *MessageBusDeployment) reconcileSecret(name types.NamespacedName, owner if equality.Semantic.DeepDerivative(target.Data, secret.Data) { return secret, nil } - return target, r.Patch(r.ctx, target, client.StrategicMergeFrom(secret)) + return target, r.Patch(ctx, target, client.StrategicMergeFrom(secret)) } // reconcileStatefulset will reconcile the messagebus statefulset to its expected state. -func (r *MessageBusDeployment) reconcileStatefulset(name types.NamespacedName, owner metav1.Object) (*appsv1.StatefulSet, error) { +func (r *MessageBusDeployment) reconcileStatefulset(ctx context.Context, logger logr.Logger, name types.NamespacedName, owner metav1.Object) (*appsv1.StatefulSet, error) { target := r.statefulset(name) if err := ctrl.SetControllerReference(owner, target, r.Scheme); err != nil { return target, err } rabbitmq := &appsv1.StatefulSet{} - if err := r.Get(r.ctx, name, rabbitmq); err != nil { + if err := r.Get(ctx, name, rabbitmq); err != nil { if !apierrors.IsNotFound(err) { return rabbitmq, err } if r.Deploy { - if err := r.Create(r.ctx, target); err != nil { + logger.Info("Creating a new MessageBus statefulset") + if err := r.Create(ctx, target); err != nil { return target, err } } return target, nil } else if !r.Deploy { - return nil, r.Delete(r.ctx, rabbitmq) + logger.Info("Removing the statefulset for MessageBus") + return nil, r.Delete(ctx, rabbitmq) } - return target, r.Patch(r.ctx, target, client.StrategicMergeFrom(rabbitmq)) + return target, r.Patch(ctx, target, client.StrategicMergeFrom(rabbitmq)) } // reconcileService will reconcile the messagebus service to its expected state. -func (r *MessageBusDeployment) reconcileService(name types.NamespacedName, owner metav1.Object) (*corev1.Service, error) { +func (r *MessageBusDeployment) reconcileService(ctx context.Context, logger logr.Logger, name types.NamespacedName, owner metav1.Object) (*corev1.Service, error) { target := r.service(name) if err := ctrl.SetControllerReference(owner, target, r.Scheme); err != nil { return target, err } service := &corev1.Service{} - if err := r.Get(r.ctx, name, service); err != nil { + if err := r.Get(ctx, name, service); err != nil { if !apierrors.IsNotFound(err) { return service, err } if r.Deploy { - if err := r.Create(r.ctx, target); err != nil { + logger.Info("Creating a new MessageBus kubernetes service") + if err := r.Create(ctx, target); err != nil { return target, err } } return target, nil } else if !r.Deploy { - return nil, r.Delete(r.ctx, service) + logger.Info("Removing the kubernetes service for MessageBus") + return nil, r.Delete(ctx, service) } - return target, r.Patch(r.ctx, target, client.StrategicMergeFrom(service)) + return target, r.Patch(ctx, target, client.StrategicMergeFrom(service)) } // secret will create a secret resource definition for the messagebus. -func (r *MessageBusDeployment) secret(name types.NamespacedName) (*corev1.Secret, error) { - data, err := r.secretData(name.Namespace) +func (r *MessageBusDeployment) secret(ctx context.Context, name types.NamespacedName) (*corev1.Secret, error) { + data, err := r.secretData(ctx, name.Namespace) if err != nil { return nil, err } @@ -195,7 +202,7 @@ func (r *MessageBusDeployment) service(name types.NamespacedName) *corev1.Servic } // secretData will create a map of secrets for the messagebus secret. -func (r *MessageBusDeployment) secretData(namespace string) (map[string][]byte, error) { +func (r *MessageBusDeployment) secretData(ctx context.Context, namespace string) (map[string][]byte, error) { data := map[string][]byte{ "ETOS_RABBITMQ_HOST": []byte(r.Host), "ETOS_RABBITMQ_EXCHANGE": []byte(r.Exchange), @@ -205,7 +212,7 @@ func (r *MessageBusDeployment) secretData(namespace string) (map[string][]byte, } if r.Password != nil { - password, err := r.Password.Get(r.ctx, r.Client, namespace) + password, err := r.Password.Get(ctx, r.Client, namespace) if err != nil { return nil, err } diff --git a/internal/extras/mongodb.go b/internal/extras/mongodb.go index 8aab481e..e8c701d2 100644 --- a/internal/extras/mongodb.go +++ b/internal/extras/mongodb.go @@ -21,6 +21,7 @@ import ( "net/url" etosv1alpha1 "github.com/eiffel-community/etos/api/v1alpha1" + "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" @@ -52,8 +53,8 @@ func NewMongoDBDeployment(spec etosv1alpha1.MongoDB, scheme *runtime.Scheme, cli // Reconcile will reconcile MongoDB to its expected state. func (r *MongoDBDeployment) Reconcile(ctx context.Context, cluster *etosv1alpha1.Cluster) error { - logger := log.FromContext(ctx) name := fmt.Sprintf("%s-mongodb", cluster.Name) + logger := log.FromContext(ctx, "Reconciler", "MongoDB", "BaseName", name) namespacedName := types.NamespacedName{Name: name, Namespace: cluster.Namespace} if (url.URL{}) == r.URL { @@ -72,19 +73,24 @@ func (r *MongoDBDeployment) Reconcile(ctx context.Context, cluster *etosv1alpha1 logger.Info("Patching host & port when deploying mongodb", "host", name, "port", mongodbPort) r.URL.Host = fmt.Sprintf("%s:%d", name, mongodbPort) r.URI.Value = r.URL.String() + } else { + logger.Info("Not deploying MongoDB") } - secret, err := r.reconcileSecret(ctx, namespacedName, cluster) + secret, err := r.reconcileSecret(ctx, logger, namespacedName, cluster) if err != nil { + logger.Error(err, "Failed to reconcile the MongoDB secret") return err } r.SecretName = secret.Name - _, err = r.reconcileStatefulset(ctx, namespacedName, cluster) + _, err = r.reconcileStatefulset(ctx, logger, namespacedName, cluster) if err != nil { + logger.Error(err, "Failed to reconcile the MongoDB statefulset") return err } - _, err = r.reconcileService(ctx, namespacedName, cluster) + _, err = r.reconcileService(ctx, logger, namespacedName, cluster) if err != nil { + logger.Error(err, "Failed to reconcile the MongoDB service") return err } @@ -92,8 +98,7 @@ func (r *MongoDBDeployment) Reconcile(ctx context.Context, cluster *etosv1alpha1 } // reconcileSecret will reconcile the MongoDB secret to its expected state. -func (r *MongoDBDeployment) reconcileSecret(ctx context.Context, name types.NamespacedName, owner metav1.Object) (*corev1.Secret, error) { - logger := log.FromContext(ctx) +func (r *MongoDBDeployment) reconcileSecret(ctx context.Context, logger logr.Logger, name types.NamespacedName, owner metav1.Object) (*corev1.Secret, error) { target := r.secret(name) if err := ctrl.SetControllerReference(owner, target, r.Scheme); err != nil { return target, err @@ -114,6 +119,7 @@ func (r *MongoDBDeployment) reconcileSecret(ctx context.Context, name types.Name } return target, nil } else if !r.Deploy { + logger.Info("Removing the MongoDB secret") return nil, r.Delete(ctx, secret) } if equality.Semantic.DeepDerivative(target.Data, secret.Data) { @@ -123,7 +129,7 @@ func (r *MongoDBDeployment) reconcileSecret(ctx context.Context, name types.Name } // reconcileStatefulset will reconcile the MongoDB statefulset to its expected state. -func (r *MongoDBDeployment) reconcileStatefulset(ctx context.Context, name types.NamespacedName, owner metav1.Object) (*appsv1.StatefulSet, error) { +func (r *MongoDBDeployment) reconcileStatefulset(ctx context.Context, logger logr.Logger, name types.NamespacedName, owner metav1.Object) (*appsv1.StatefulSet, error) { target := r.statefulset(name) if err := ctrl.SetControllerReference(owner, target, r.Scheme); err != nil { return target, err @@ -135,19 +141,21 @@ func (r *MongoDBDeployment) reconcileStatefulset(ctx context.Context, name types return mongodb, err } if r.Deploy { + logger.Info("Deploying a new MongoDB statefulset") if err := r.Create(ctx, target); err != nil { return target, err } } return mongodb, nil } else if !r.Deploy { + logger.Info("Removing the MongoDB statefulset") return nil, r.Delete(ctx, mongodb) } return target, r.Patch(ctx, target, client.StrategicMergeFrom(mongodb)) } // reconcileService will reconcile the MongoDB service to its expected state. -func (r *MongoDBDeployment) reconcileService(ctx context.Context, name types.NamespacedName, owner metav1.Object) (*corev1.Service, error) { +func (r *MongoDBDeployment) reconcileService(ctx context.Context, logger logr.Logger, name types.NamespacedName, owner metav1.Object) (*corev1.Service, error) { target := r.service(name) if err := ctrl.SetControllerReference(owner, target, r.Scheme); err != nil { return target, err @@ -159,12 +167,14 @@ func (r *MongoDBDeployment) reconcileService(ctx context.Context, name types.Nam return service, err } if r.Deploy { + logger.Info("Creating a new MongoDB kubernetes service") if err := r.Create(ctx, target); err != nil { return target, err } } return service, nil } else if !r.Deploy { + logger.Info("Removing the kubernetes service for MongoDB") return nil, r.Delete(ctx, service) } return target, r.Patch(ctx, target, client.StrategicMergeFrom(service)) diff --git a/internal/extras/rabbitmq.go b/internal/extras/rabbitmq.go index abd262f9..ee81e363 100644 --- a/internal/extras/rabbitmq.go +++ b/internal/extras/rabbitmq.go @@ -20,6 +20,7 @@ import ( "fmt" etosv1alpha1 "github.com/eiffel-community/etos/api/v1alpha1" + "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" @@ -39,21 +40,19 @@ var rabbitmqPort int32 = 5672 type RabbitMQDeployment struct { etosv1alpha1.RabbitMQ client.Client - ctx context.Context Scheme *runtime.Scheme SecretName string } // NewRabbitMQDeployment will create a new RabbitMQ reconciler. func NewRabbitMQDeployment(spec etosv1alpha1.RabbitMQ, scheme *runtime.Scheme, client client.Client) *RabbitMQDeployment { - return &RabbitMQDeployment{spec, client, nil, scheme, ""} + return &RabbitMQDeployment{spec, client, scheme, ""} } // Reconcile will reconcile RabbitMQ to its expected state. func (r *RabbitMQDeployment) Reconcile(ctx context.Context, cluster *etosv1alpha1.Cluster) error { - logger := log.FromContext(ctx) - r.ctx = ctx name := fmt.Sprintf("%s-rabbitmq", cluster.Name) + logger := log.FromContext(ctx, "Reconciler", "RabbitMQ", "BaseName", name) namespacedName := types.NamespacedName{Name: name, Namespace: cluster.Namespace} if r.Deploy { logger.Info("Patching host & port when deploying RabbitMQ", "host", name, "port", rabbitmqPort) @@ -61,18 +60,21 @@ func (r *RabbitMQDeployment) Reconcile(ctx context.Context, cluster *etosv1alpha r.Port = fmt.Sprintf("%d", rabbitmqPort) } - secret, err := r.reconcileSecret(namespacedName, cluster) + secret, err := r.reconcileSecret(ctx, logger, namespacedName, cluster) if err != nil { + logger.Error(err, "Failed to reconcile the RabbitMQ secret") return err } r.SecretName = secret.Name - _, err = r.reconcileStatefulset(namespacedName, cluster) + _, err = r.reconcileStatefulset(ctx, logger, namespacedName, cluster) if err != nil { + logger.Error(err, "Failed to reconcile the RabbitMQ statefulset") return err } - _, err = r.reconcileService(namespacedName, cluster) + _, err = r.reconcileService(ctx, logger, namespacedName, cluster) if err != nil { + logger.Error(err, "Failed to reconcile the RabbitMQ service") return err } @@ -80,9 +82,8 @@ func (r *RabbitMQDeployment) Reconcile(ctx context.Context, cluster *etosv1alpha } // reconcileSecret will reconcile the RabbitMQ secret to its expected state. -func (r *RabbitMQDeployment) reconcileSecret(name types.NamespacedName, owner metav1.Object) (*corev1.Secret, error) { - logger := log.FromContext(r.ctx) - target, err := r.secret(name) +func (r *RabbitMQDeployment) reconcileSecret(ctx context.Context, logger logr.Logger, name types.NamespacedName, owner metav1.Object) (*corev1.Secret, error) { + target, err := r.secret(ctx, name) if err != nil { return target, err } @@ -92,13 +93,13 @@ func (r *RabbitMQDeployment) reconcileSecret(name types.NamespacedName, owner me scheme.Scheme.Default(target) secret := &corev1.Secret{} - if err := r.Get(r.ctx, name, secret); err != nil { + if err := r.Get(ctx, name, secret); err != nil { if !apierrors.IsNotFound(err) { logger.Error(err, "failed to get rabbitmq secret") return secret, err } logger.Info("Secret not found. Creating") - if err := r.Create(r.ctx, target); err != nil { + if err := r.Create(ctx, target); err != nil { return target, err } return target, nil @@ -106,60 +107,64 @@ func (r *RabbitMQDeployment) reconcileSecret(name types.NamespacedName, owner me if equality.Semantic.DeepDerivative(target.Data, secret.Data) { return secret, nil } - return target, r.Patch(r.ctx, target, client.StrategicMergeFrom(secret)) + return target, r.Patch(ctx, target, client.StrategicMergeFrom(secret)) } // reconcileStatefulset will reconcile the RabbitMQ statefulset to its expected state. -func (r *RabbitMQDeployment) reconcileStatefulset(name types.NamespacedName, owner metav1.Object) (*appsv1.StatefulSet, error) { +func (r *RabbitMQDeployment) reconcileStatefulset(ctx context.Context, logger logr.Logger, name types.NamespacedName, owner metav1.Object) (*appsv1.StatefulSet, error) { target := r.statefulset(name) if err := ctrl.SetControllerReference(owner, target, r.Scheme); err != nil { return target, err } rabbitmq := &appsv1.StatefulSet{} - if err := r.Get(r.ctx, name, rabbitmq); err != nil { + if err := r.Get(ctx, name, rabbitmq); err != nil { if !apierrors.IsNotFound(err) { return rabbitmq, err } if r.Deploy { - if err := r.Create(r.ctx, target); err != nil { + logger.Info("Creating a new RabbitMQ statefulset") + if err := r.Create(ctx, target); err != nil { return target, err } } return target, nil } else if !r.Deploy { - return nil, r.Delete(r.ctx, rabbitmq) + logger.Info("Removing the statefulset for RabbitMQ") + return nil, r.Delete(ctx, rabbitmq) } - return target, r.Patch(r.ctx, target, client.StrategicMergeFrom(rabbitmq)) + return target, r.Patch(ctx, target, client.StrategicMergeFrom(rabbitmq)) } // reconcileService will reconcile the RabbitMQ service to its expected state. -func (r *RabbitMQDeployment) reconcileService(name types.NamespacedName, owner metav1.Object) (*corev1.Service, error) { +func (r *RabbitMQDeployment) reconcileService(ctx context.Context, logger logr.Logger, name types.NamespacedName, owner metav1.Object) (*corev1.Service, error) { target := r.service(name) if err := ctrl.SetControllerReference(owner, target, r.Scheme); err != nil { return target, err } service := &corev1.Service{} - if err := r.Get(r.ctx, name, service); err != nil { + if err := r.Get(ctx, name, service); err != nil { if !apierrors.IsNotFound(err) { return service, err } if r.Deploy { - if err := r.Create(r.ctx, target); err != nil { + logger.Info("Creating a new RabbitMQ kubernetes service") + if err := r.Create(ctx, target); err != nil { return target, err } } return target, nil } else if !r.Deploy { - return nil, r.Delete(r.ctx, service) + logger.Info("Removing the kubernetes service for RabbitMQ") + return nil, r.Delete(ctx, service) } - return target, r.Patch(r.ctx, target, client.StrategicMergeFrom(service)) + return target, r.Patch(ctx, target, client.StrategicMergeFrom(service)) } // secret will create a secret resource definition for RabbitMQ. -func (r *RabbitMQDeployment) secret(name types.NamespacedName) (*corev1.Secret, error) { - data, err := r.secretData(name) +func (r *RabbitMQDeployment) secret(ctx context.Context, name types.NamespacedName) (*corev1.Secret, error) { + data, err := r.secretData(ctx, name) if err != nil { return nil, err } @@ -201,7 +206,7 @@ func (r *RabbitMQDeployment) service(name types.NamespacedName) *corev1.Service } // secretData will create a map of secrets for the RabbitMQ secret. -func (r *RabbitMQDeployment) secretData(name types.NamespacedName) (map[string][]byte, error) { +func (r *RabbitMQDeployment) secretData(ctx context.Context, name types.NamespacedName) (map[string][]byte, error) { data := map[string][]byte{ "RABBITMQ_HOST": []byte(r.Host), "RABBITMQ_EXCHANGE": []byte(r.Exchange), @@ -210,7 +215,7 @@ func (r *RabbitMQDeployment) secretData(name types.NamespacedName) (map[string][ "RABBITMQ_VHOST": []byte(r.Vhost), } if r.Password != nil { - password, err := r.Password.Get(r.ctx, r.Client, name.Namespace) + password, err := r.Password.Get(ctx, r.Client, name.Namespace) if err != nil { return nil, err } From aaf133f6740dd87aec46732b0e7ac739fb59d949 Mon Sep 17 00:00:00 2001 From: Tobias Persson Date: Mon, 21 Oct 2024 11:48:55 +0200 Subject: [PATCH 07/11] Don't deploy the event repository config if deploy is false --- internal/extras/eventrepository.go | 37 +++++++++++++++++++++--------- internal/extras/mongodb.go | 2 +- 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/internal/extras/eventrepository.go b/internal/extras/eventrepository.go index 0a351eae..63fd4594 100644 --- a/internal/extras/eventrepository.go +++ b/internal/extras/eventrepository.go @@ -64,8 +64,14 @@ func (r *EventRepositoryDeployment) Reconcile(ctx context.Context, cluster *etos logger.Error(err, "Failed to reconcile the EventRepository configuration") return err } + var configName string + if cfg != nil { + configName = cfg.ObjectMeta.Name + } else { + configName = namespacedName.Name + } - _, err = r.reconcileDeployment(ctx, logger, namespacedName, cfg.ObjectMeta.Name, cluster) + _, err = r.reconcileDeployment(ctx, logger, namespacedName, configName, cluster) if err != nil { logger.Error(err, "Failed to reconcile the EventRepository deployment") return err @@ -93,24 +99,33 @@ func (r *EventRepositoryDeployment) Reconcile(ctx context.Context, cluster *etos // reconcileConfig will reconcile the secret to use as configuration for the event repository. func (r *EventRepositoryDeployment) reconcileConfig(ctx context.Context, logger logr.Logger, name types.NamespacedName, owner metav1.Object) (*corev1.Secret, error) { - target, err := r.config(ctx, name) - if err != nil { - return nil, err - } - if err := ctrl.SetControllerReference(owner, target, r.Scheme); err != nil { - return target, err + var err error + var target *corev1.Secret + if r.Deploy { + target, err = r.config(ctx, name) + if err != nil { + return nil, err + } + if err = ctrl.SetControllerReference(owner, target, r.Scheme); err != nil { + return target, err + } } secret := &corev1.Secret{} - if err := r.Get(ctx, name, secret); err != nil { + if err = r.Get(ctx, name, secret); err != nil { if !apierrors.IsNotFound(err) { return secret, err } - logger.Info("Creating the configuration for an EventRepository") - if err := r.Create(ctx, target); err != nil { - return target, err + if r.Deploy { + logger.Info("Creating the configuration for an EventRepository") + if err = r.Create(ctx, target); err != nil { + return target, err + } } return target, nil + } else if !r.Deploy { + logger.Info("Removing the configuration for EventRepository") + return nil, r.Delete(ctx, secret) } return target, r.Patch(ctx, target, client.StrategicMergeFrom(secret)) } diff --git a/internal/extras/mongodb.go b/internal/extras/mongodb.go index e8c701d2..9402e32d 100644 --- a/internal/extras/mongodb.go +++ b/internal/extras/mongodb.go @@ -111,8 +111,8 @@ func (r *MongoDBDeployment) reconcileSecret(ctx context.Context, logger logr.Log logger.Error(err, "failed to get mongodb secret") return secret, err } - logger.Info("Secret not found. Creating") if r.Deploy { + logger.Info("Secret not found. Creating") if err := r.Create(ctx, target); err != nil { return target, err } From cb0348cdc60f17e5554a93237053da45eef2689d Mon Sep 17 00:00:00 2001 From: Tobias Persson Date: Mon, 21 Oct 2024 12:25:00 +0200 Subject: [PATCH 08/11] Set the ETOS_RABBITMQ_QUEUE for the suite runner --- internal/etos/etos.go | 6 +++--- internal/etos/suitestarter/suitestarter.go | 9 +++++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/internal/etos/etos.go b/internal/etos/etos.go index 8bda18f1..91ffc4fb 100644 --- a/internal/etos/etos.go +++ b/internal/etos/etos.go @@ -370,11 +370,11 @@ func (r *ETOSDeployment) config(name types.NamespacedName, cluster *etosv1alpha1 "SOURCE_HOST": []byte(r.Config.Source), "ETOS_API": []byte(etosApi), "SUITE_RUNNER_IMAGE": []byte(cluster.Spec.ETOS.SuiteRunner.Image.Image), - "SUITE_RUNNER_IMAGE_PULL_POLICY": []byte(string(cluster.Spec.ETOS.SuiteRunner.ImagePullPolicy)), + "SUITE_RUNNER_IMAGE_PULL_POLICY": []byte(cluster.Spec.ETOS.SuiteRunner.ImagePullPolicy), "LOG_LISTENER_IMAGE": []byte(cluster.Spec.ETOS.SuiteRunner.LogListener.Image.Image), - "LOG_LISTENER_IMAGE_PULL_POLICY": []byte(string(cluster.Spec.ETOS.SuiteRunner.LogListener.ImagePullPolicy)), + "LOG_LISTENER_IMAGE_PULL_POLICY": []byte(cluster.Spec.ETOS.SuiteRunner.LogListener.ImagePullPolicy), "ENVIRONMENT_PROVIDER_IMAGE": []byte(cluster.Spec.ETOS.EnvironmentProvider.Image.Image), - "ENVIRONMENT_PROVIDER_IMAGE_PULL_POLICY": []byte(string(cluster.Spec.ETOS.EnvironmentProvider.ImagePullPolicy)), + "ENVIRONMENT_PROVIDER_IMAGE_PULL_POLICY": []byte(cluster.Spec.ETOS.EnvironmentProvider.ImagePullPolicy), "ETR_VERSION": []byte(cluster.Spec.ETOS.TestRunner.Version), "ETOS_ROUTING_KEY_TAG": []byte(cluster.Spec.ETOS.Config.RoutingKeyTag), diff --git a/internal/etos/suitestarter/suitestarter.go b/internal/etos/suitestarter/suitestarter.go index d6e98281..ac51948c 100644 --- a/internal/etos/suitestarter/suitestarter.go +++ b/internal/etos/suitestarter/suitestarter.go @@ -140,7 +140,7 @@ func (r *ETOSSuiteStarterDeployment) reconcileSuiteRunnerServiceAccount(ctx cont // reconcileSuiteRunnerSecret will reconcile the ETOS suite runner secret to its expected state. func (r *ETOSSuiteStarterDeployment) reconcileSuiteRunnerSecret(ctx context.Context, logger logr.Logger, name types.NamespacedName, cluster *etosv1alpha1.Cluster) (*corev1.Secret, error) { name.Name = fmt.Sprintf("%s-etos-suite-runner-cfg", cluster.ObjectMeta.Name) - target, err := r.mergedSecret(ctx, name) + target, err := r.mergedSecret(ctx, name, cluster) if err != nil { return nil, err } @@ -384,7 +384,7 @@ func (r *ETOSSuiteStarterDeployment) secret(name, serviceAccountName types.Names // mergedSecret creates a secret that is the merged values of Eiffel, ETOS and encryption key secrets. // This is for use in the suite runner which only has a single secret as input. -func (r *ETOSSuiteStarterDeployment) mergedSecret(ctx context.Context, name types.NamespacedName) (*corev1.Secret, error) { +func (r *ETOSSuiteStarterDeployment) mergedSecret(ctx context.Context, name types.NamespacedName, cluster *etosv1alpha1.Cluster) (*corev1.Secret, error) { eiffel := &corev1.Secret{} if err := r.Get(ctx, types.NamespacedName{Name: r.rabbitmqSecret, Namespace: name.Namespace}, eiffel); err != nil { return nil, err @@ -398,6 +398,11 @@ func (r *ETOSSuiteStarterDeployment) mergedSecret(ctx context.Context, name type maps.Copy(data, etos.Data) maps.Copy(data, r.etosConfig.Data) maps.Copy(data, r.encryptionSecret.Data) + // Used by the LogListener and the CreateQueue initContainer. + data["ETOS_RABBITMQ_QUEUE_NAME"] = []byte(cluster.Spec.ETOS.SuiteRunner.LogListener.ETOSQueueName) + if cluster.Spec.ETOS.SuiteRunner.LogListener.ETOSQueueParams != "" { + data["ETOS_RABBITMQ_QUEUE_PARAMS"] = []byte(cluster.Spec.ETOS.SuiteRunner.LogListener.ETOSQueueParams) + } return &corev1.Secret{ ObjectMeta: r.meta(name), Data: data, From 0b3953d95d2263c27123e227018fed8649fa1f23 Mon Sep 17 00:00:00 2001 From: Tobias Persson Date: Thu, 24 Oct 2024 12:34:09 +0200 Subject: [PATCH 09/11] Name suite starter service account differently --- internal/etos/suitestarter/suitestarter.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/internal/etos/suitestarter/suitestarter.go b/internal/etos/suitestarter/suitestarter.go index ac51948c..800644b8 100644 --- a/internal/etos/suitestarter/suitestarter.go +++ b/internal/etos/suitestarter/suitestarter.go @@ -59,8 +59,7 @@ func (r *ETOSSuiteStarterDeployment) Reconcile(ctx context.Context, cluster *eto name := fmt.Sprintf("%s-etos-suite-starter", cluster.Name) logger := log.FromContext(ctx, "Reconciler", "ETOSSuiteStarter", "BaseName", name) namespacedName := types.NamespacedName{Name: name, Namespace: cluster.Namespace} - // There is an assumption in the suite starter that the service account is named 'etos-sa'. - _, err = r.reconcileSuiteRunnerServiceAccount(ctx, logger, types.NamespacedName{Name: "etos-sa", Namespace: cluster.Namespace}, cluster) + _, err = r.reconcileSuiteRunnerServiceAccount(ctx, logger, namespacedName, cluster) if err != nil { logger.Error(err, "Failed to reconcile the Suite runner service account") return err @@ -535,7 +534,7 @@ func (r *ETOSSuiteStarterDeployment) container(name types.NamespacedName, secret // suiteRunnerTemplate creates a secret resource for the ETOS SuiteStarter. func (r *ETOSSuiteStarterDeployment) suiteRunnerTemplate(name types.NamespacedName) *corev1.Secret { data := map[string][]byte{ - "suite_runner_template.yaml": []byte(` + "suite_runner_template.yaml": []byte(fmt.Sprintf(` apiVersion: batch/v1 kind: Job metadata: @@ -574,7 +573,7 @@ func (r *ETOSSuiteStarterDeployment) suiteRunnerTemplate(name types.NamespacedNa env: - name: TERCC value: '{EiffelTestExecutionRecipeCollectionCreatedEvent}' - serviceAccountName: etos-sa + serviceAccountName: %s containers: - name: {job_name} image: {docker_image} @@ -632,7 +631,7 @@ func (r *ETOSSuiteStarterDeployment) suiteRunnerTemplate(name types.NamespacedNa mountPath: /kubexit restartPolicy: Never backoffLimit: 0 - `), + `, name.Name)), } return &corev1.Secret{ ObjectMeta: r.meta(name), From 9da975d2c43de65b0ca74fbc50b02c2cb311b307 Mon Sep 17 00:00:00 2001 From: Tobias Persson Date: Fri, 25 Oct 2024 14:03:30 +0200 Subject: [PATCH 10/11] Remove ttl --- internal/controller/testrun_controller.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/internal/controller/testrun_controller.go b/internal/controller/testrun_controller.go index 0530c085..3c46b219 100644 --- a/internal/controller/testrun_controller.go +++ b/internal/controller/testrun_controller.go @@ -463,7 +463,6 @@ func (r TestRunReconciler) environmentRequest(testrun *etosv1alpha1.TestRun, sui // suiteRunnerJob is the job definition for an etos suite runner. func (r TestRunReconciler) suiteRunnerJob(tercc []byte, testrun *etosv1alpha1.TestRun) *batchv1.Job { - ttl := int32(300) grace := int64(30) backoff := int32(0) return &batchv1.Job{ @@ -481,8 +480,7 @@ func (r TestRunReconciler) suiteRunnerJob(tercc []byte, testrun *etosv1alpha1.Te Namespace: testrun.Namespace, }, Spec: batchv1.JobSpec{ - TTLSecondsAfterFinished: &ttl, - BackoffLimit: &backoff, + BackoffLimit: &backoff, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Name: testrun.Name, From 880f15b944a0f2fdd977bd3fce7fceb0571ec6a5 Mon Sep 17 00:00:00 2001 From: Tobias Persson Date: Wed, 27 Nov 2024 09:45:00 +0100 Subject: [PATCH 11/11] Review comments --- api/v1alpha1/cluster_types.go | 5 +++-- api/v1alpha1/environment_types.go | 1 + api/v1alpha1/environmentrequest_types.go | 2 +- api/v1alpha1/testrun_types.go | 4 ++-- .../crd/bases/etos.eiffel-community.github.io_clusters.yaml | 6 ++++-- ...etos.eiffel-community.github.io_environmentrequests.yaml | 3 ++- .../crd/bases/etos.eiffel-community.github.io_testruns.yaml | 5 +++-- 7 files changed, 16 insertions(+), 10 deletions(-) diff --git a/api/v1alpha1/cluster_types.go b/api/v1alpha1/cluster_types.go index 4107b45b..9feb1991 100644 --- a/api/v1alpha1/cluster_types.go +++ b/api/v1alpha1/cluster_types.go @@ -100,7 +100,8 @@ type Database struct { // ETOSAPI describes the deployment of the ETOS API. type ETOSAPI struct { Image `json:",inline"` - // The provider secrets are necessary in order to run ETOS the old way and not using the controller. + // The provider secrets are necessary in order deploy and run ETOS without using the + // kubernetes controller. // They can be removed from here when the suite starter is no longer in use. // +optional IUTProviderSecret string `json:"iutProviderSecret"` @@ -149,7 +150,7 @@ type ETOSSuiteStarter struct { Config ETOSSuiteStarterConfig `json:"config"` } -// ETOSSSE describes th deployment of an ETOS SSE API. +// ETOSSSE describes th deployment of an ETOS Server Sent Events API. type ETOSSSE struct { Image `json:",inline"` } diff --git a/api/v1alpha1/environment_types.go b/api/v1alpha1/environment_types.go index 0ae2a324..2bf0ca6b 100644 --- a/api/v1alpha1/environment_types.go +++ b/api/v1alpha1/environment_types.go @@ -26,6 +26,7 @@ type EnvironmentSpec struct { Name string `json:"name"` // Snake casing as to be compatible with ETR. + // Regexes match any UUID version. // +kubebuilder:validation:Pattern="^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$" SuiteID string `json:"suite_id"` diff --git a/api/v1alpha1/environmentrequest_types.go b/api/v1alpha1/environmentrequest_types.go index 8fd23c7c..0c6f7247 100644 --- a/api/v1alpha1/environmentrequest_types.go +++ b/api/v1alpha1/environmentrequest_types.go @@ -47,7 +47,7 @@ type Splitter struct { // EnvironmentRequestSpec defines the desired state of EnvironmentRequest type EnvironmentRequestSpec struct { - // ID is the ID for the environments generated. Will be generated if nil + // ID is the ID for the environments generated. Will be generated if nil. The ID is a UUID, any version, and regex matches that. // +kubebuilder:validation:Pattern="^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$" ID string `json:"id,omitempty"` Name string `json:"name,omitempty"` diff --git a/api/v1alpha1/testrun_types.go b/api/v1alpha1/testrun_types.go index 6c1b0688..004aaa19 100644 --- a/api/v1alpha1/testrun_types.go +++ b/api/v1alpha1/testrun_types.go @@ -103,11 +103,11 @@ type TestRunSpec struct { // Name of the ETOS cluster to execute the testrun in. Cluster string `json:"cluster,omitempty"` - // ID is the test suite ID for this execution. Will be generated if nil + // ID is the test suite ID for this execution. Will be generated if nil. The ID is a UUID, any version, and regex matches that. // +kubebuilder:validation:Pattern="^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$" ID string `json:"id,omitempty"` - // Artifact is the ID of the software under test. + // Artifact is the ID of the software under test. The ID is a UUID, any version, and regex matches that. // +kubebuilder:validation:Pattern="^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$" Artifact string `json:"artifact"` diff --git a/config/crd/bases/etos.eiffel-community.github.io_clusters.yaml b/config/crd/bases/etos.eiffel-community.github.io_clusters.yaml index 5c871e80..90a0c488 100644 --- a/config/crd/bases/etos.eiffel-community.github.io_clusters.yaml +++ b/config/crd/bases/etos.eiffel-community.github.io_clusters.yaml @@ -87,7 +87,8 @@ spec: type: string iutProviderSecret: description: |- - The provider secrets are necessary in order to run ETOS the old way and not using the controller. + The provider secrets are necessary in order deploy and run ETOS without using the + kubernetes controller. They can be removed from here when the suite starter is no longer in use. type: string logAreaProviderSecret: @@ -254,7 +255,8 @@ spec: sse: default: image: registry.nordix.org/eiffel/etos-sse:latest - description: ETOSSSE describes th deployment of an ETOS SSE API. + description: ETOSSSE describes th deployment of an ETOS Server + Sent Events API. properties: image: type: string diff --git a/config/crd/bases/etos.eiffel-community.github.io_environmentrequests.yaml b/config/crd/bases/etos.eiffel-community.github.io_environmentrequests.yaml index 90cb1b19..467bd8f7 100644 --- a/config/crd/bases/etos.eiffel-community.github.io_environmentrequests.yaml +++ b/config/crd/bases/etos.eiffel-community.github.io_environmentrequests.yaml @@ -57,7 +57,8 @@ spec: x-kubernetes-preserve-unknown-fields: true id: description: ID is the ID for the environments generated. Will be - generated if nil + generated if nil. The ID is a UUID, any version, and regex matches + that. pattern: ^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$ type: string identifier: diff --git a/config/crd/bases/etos.eiffel-community.github.io_testruns.yaml b/config/crd/bases/etos.eiffel-community.github.io_testruns.yaml index d35f7211..e4f2a329 100644 --- a/config/crd/bases/etos.eiffel-community.github.io_testruns.yaml +++ b/config/crd/bases/etos.eiffel-community.github.io_testruns.yaml @@ -62,7 +62,8 @@ spec: description: TestRunSpec defines the desired state of TestRun properties: artifact: - description: Artifact is the ID of the software under test. + description: Artifact is the ID of the software under test. The ID + is a UUID, any version, and regex matches that. pattern: ^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$ type: string cluster: @@ -82,7 +83,7 @@ spec: type: object id: description: ID is the test suite ID for this execution. Will be generated - if nil + if nil. The ID is a UUID, any version, and regex matches that. pattern: ^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$ type: string identity: