From 9f5e97b2b2162497f70e3654e7e05f031adfcc82 Mon Sep 17 00:00:00 2001 From: Pavlo Kostianov Date: Fri, 29 May 2026 05:56:45 -0400 Subject: [PATCH] Add configurable logging for OpenStack Lightspeed components Add a logging subsection to the OpenStackLightspeed CRD with three configurable log level fields for independent control of each container. Changes: - Add LoggingConfig struct with three fields: - ogxLogLevel: Controls OGX/llama-stack container - Supports standard levels (DEBUG, INFO) OR fine-grained syntax ("core=debug,providers=info") - Defaults to "all=info" if empty - lightspeedStackLogLevel: Controls lightspeed-service-api container - Enum: DEBUG, INFO, WARNING, ERROR, CRITICAL - Defaults to INFO - dataverseExporterLogLevel: Controls dataverse exporter sidecar - Enum: DEBUG, INFO, WARNING, ERROR, CRITICAL - Defaults to INFO - Embed LoggingConfig under 'logging:' subsection in CRD spec - Update KUTTL tests to verify environment variables and container args Independent fields enable targeted debugging without flooding logs from all containers. Fine-grained OGX logging supports component-level control for troubleshooting specific subsystems. Co-Authored-By: Claude Sonnet 4.5 --- api/v1beta1/openstacklightspeed_types.go | 26 ++++++++++ api/v1beta1/zz_generated.deepcopy.go | 16 +++++++ ...ed.openstack.org_openstacklightspeeds.yaml | 34 ++++++++++++++ ...tspeed-operator.clusterserviceversion.yaml | 15 +++++- ...ed.openstack.org_openstacklightspeeds.yaml | 34 ++++++++++++++ ...tspeed-operator.clusterserviceversion.yaml | 13 +++++ internal/controller/lcore_deployment.go | 43 ++++++++++++++--- .../assert-openstack-lightspeed-instance.yaml | 47 +++++++++++++++++-- .../create-openstack-lightspeed-instance.yaml | 4 ++ ...-update-openstack-lightspeed-instance.yaml | 4 ++ .../08-assert-openstacklightspeed-update.yaml | 38 ++++++++++++++- 11 files changed, 261 insertions(+), 13 deletions(-) diff --git a/api/v1beta1/openstacklightspeed_types.go b/api/v1beta1/openstacklightspeed_types.go index d8f6f028..275a3ed0 100644 --- a/api/v1beta1/openstacklightspeed_types.go +++ b/api/v1beta1/openstacklightspeed_types.go @@ -86,6 +86,28 @@ type OpenStackLightspeedSpec struct { Database *DatabaseSpec `json:"database,omitempty"` } +// LoggingConfig defines logging configuration for OpenStackLightspeed components +type LoggingConfig struct { + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="OGX Log Level" + // Log level configuration for the OGX/llama-stack container. Supports standard levels (INFO, DEBUG) or fine-grained control using format "component=level,component=level" (e.g., "core=debug,providers=info"). Defaults to "all=info" if empty. + OGXLogLevel string `json:"ogxLogLevel,omitempty"` + + // +kubebuilder:validation:Optional + // +kubebuilder:validation:Enum=DEBUG;INFO;WARNING;ERROR;CRITICAL + // +kubebuilder:default="INFO" + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Lightspeed Stack Log Level" + // Log level for the lightspeed-service-api container. Supports standard Python log levels: DEBUG, INFO, WARNING, ERROR, CRITICAL. + LightspeedStackLogLevel string `json:"lightspeedStackLogLevel,omitempty"` + + // +kubebuilder:validation:Optional + // +kubebuilder:validation:Enum=DEBUG;INFO;WARNING;ERROR;CRITICAL + // +kubebuilder:default="INFO" + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Dataverse Exporter Log Level" + // Log level for the dataverse exporter sidecar container. Supports standard Python log levels: DEBUG, INFO, WARNING, ERROR, CRITICAL. + DataverseExporterLogLevel string `json:"dataverseExporterLogLevel,omitempty"` +} + // OpenStackLightspeedCore defines the desired state of OpenStackLightspeed type OpenStackLightspeedCore struct { // +kubebuilder:validation:Required @@ -138,6 +160,10 @@ type OpenStackLightspeedCore struct { // +kubebuilder:validation:Optional // Disable conversation transcripts collection TranscriptsDisabled bool `json:"transcriptsDisabled,omitempty"` + + // +kubebuilder:validation:Optional + // Logging configuration for OpenStackLightspeed components + Logging LoggingConfig `json:"logging,omitempty"` } // OpenStackLightspeedStatus defines the observed state of OpenStackLightspeed diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 80eefb2e..c74fc454 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -41,6 +41,21 @@ func (in *DatabaseSpec) DeepCopy() *DatabaseSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LoggingConfig) DeepCopyInto(out *LoggingConfig) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LoggingConfig. +func (in *LoggingConfig) DeepCopy() *LoggingConfig { + if in == nil { + return nil + } + out := new(LoggingConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OpenStackLightspeed) DeepCopyInto(out *OpenStackLightspeed) { *out = *in @@ -71,6 +86,7 @@ func (in *OpenStackLightspeed) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OpenStackLightspeedCore) DeepCopyInto(out *OpenStackLightspeedCore) { *out = *in + out.Logging = in.Logging } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenStackLightspeedCore. diff --git a/bundle/manifests/lightspeed.openstack.org_openstacklightspeeds.yaml b/bundle/manifests/lightspeed.openstack.org_openstacklightspeeds.yaml index fc6dbfbb..8ced6397 100644 --- a/bundle/manifests/lightspeed.openstack.org_openstacklightspeeds.yaml +++ b/bundle/manifests/lightspeed.openstack.org_openstacklightspeeds.yaml @@ -107,6 +107,40 @@ spec: llmProjectID: description: Project ID for LLM providers that require it (e.g., WatsonX) type: string + logging: + description: Logging configuration for OpenStackLightspeed components + properties: + dataverseExporterLogLevel: + default: INFO + description: 'Log level for the dataverse exporter sidecar container. + Supports standard Python log levels: DEBUG, INFO, WARNING, ERROR, + CRITICAL.' + enum: + - DEBUG + - INFO + - WARNING + - ERROR + - CRITICAL + type: string + lightspeedStackLogLevel: + default: INFO + description: 'Log level for the lightspeed-service-api container. + Supports standard Python log levels: DEBUG, INFO, WARNING, ERROR, + CRITICAL.' + enum: + - DEBUG + - INFO + - WARNING + - ERROR + - CRITICAL + type: string + ogxLogLevel: + description: Log level configuration for the OGX/llama-stack container. + Supports standard levels (INFO, DEBUG) or fine-grained control + using format "component=level,component=level" (e.g., "core=debug,providers=info"). + Defaults to "all=info" if empty. + type: string + type: object maxTokensForResponse: description: MaxTokensForResponse defines the maximum number of tokens to be used for the response generation diff --git a/bundle/manifests/openstack-lightspeed-operator.clusterserviceversion.yaml b/bundle/manifests/openstack-lightspeed-operator.clusterserviceversion.yaml index 4c1d03bf..1f090c39 100644 --- a/bundle/manifests/openstack-lightspeed-operator.clusterserviceversion.yaml +++ b/bundle/manifests/openstack-lightspeed-operator.clusterserviceversion.yaml @@ -25,7 +25,7 @@ metadata: ] capabilities: Basic Install categories: AI/Machine Learning - createdAt: "2026-05-27T13:26:52Z" + createdAt: "2026-05-28T16:28:23Z" description: AI-powered virtual assistant for Red Hat OpenStack Services on OpenShift features.operators.openshift.io/cnf: "false" features.operators.openshift.io/cni: "false" @@ -128,6 +128,19 @@ spec: - description: Type of the provider serving the LLM displayName: Provider Type path: llmEndpointType + - description: 'Log level for the dataverse exporter sidecar container. Supports + standard Python log levels: DEBUG, INFO, WARNING, ERROR, CRITICAL.' + displayName: Dataverse Exporter Log Level + path: logging.dataverseExporterLogLevel + - description: 'Log level for the lightspeed-service-api container. Supports + standard Python log levels: DEBUG, INFO, WARNING, ERROR, CRITICAL.' + displayName: Lightspeed Stack Log Level + path: logging.lightspeedStackLogLevel + - description: Log level configuration for the OGX/llama-stack container. Supports + standard levels (INFO, DEBUG) or fine-grained control using format "component=level,component=level" + (e.g., "core=debug,providers=info"). Defaults to "all=info" if empty. + displayName: OGX Log Level + path: logging.ogxLogLevel - description: Name of the model to use at the API endpoint provided in LLMEndpoint displayName: Model Name path: modelName diff --git a/config/crd/bases/lightspeed.openstack.org_openstacklightspeeds.yaml b/config/crd/bases/lightspeed.openstack.org_openstacklightspeeds.yaml index b88cbcba..3ea11159 100644 --- a/config/crd/bases/lightspeed.openstack.org_openstacklightspeeds.yaml +++ b/config/crd/bases/lightspeed.openstack.org_openstacklightspeeds.yaml @@ -107,6 +107,40 @@ spec: llmProjectID: description: Project ID for LLM providers that require it (e.g., WatsonX) type: string + logging: + description: Logging configuration for OpenStackLightspeed components + properties: + dataverseExporterLogLevel: + default: INFO + description: 'Log level for the dataverse exporter sidecar container. + Supports standard Python log levels: DEBUG, INFO, WARNING, ERROR, + CRITICAL.' + enum: + - DEBUG + - INFO + - WARNING + - ERROR + - CRITICAL + type: string + lightspeedStackLogLevel: + default: INFO + description: 'Log level for the lightspeed-service-api container. + Supports standard Python log levels: DEBUG, INFO, WARNING, ERROR, + CRITICAL.' + enum: + - DEBUG + - INFO + - WARNING + - ERROR + - CRITICAL + type: string + ogxLogLevel: + description: Log level configuration for the OGX/llama-stack container. + Supports standard levels (INFO, DEBUG) or fine-grained control + using format "component=level,component=level" (e.g., "core=debug,providers=info"). + Defaults to "all=info" if empty. + type: string + type: object maxTokensForResponse: description: MaxTokensForResponse defines the maximum number of tokens to be used for the response generation diff --git a/config/manifests/bases/openstack-lightspeed-operator.clusterserviceversion.yaml b/config/manifests/bases/openstack-lightspeed-operator.clusterserviceversion.yaml index 25582f02..fb5cda1a 100644 --- a/config/manifests/bases/openstack-lightspeed-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/openstack-lightspeed-operator.clusterserviceversion.yaml @@ -105,6 +105,19 @@ spec: - description: Type of the provider serving the LLM displayName: Provider Type path: llmEndpointType + - description: 'Log level for the dataverse exporter sidecar container. Supports + standard Python log levels: DEBUG, INFO, WARNING, ERROR, CRITICAL.' + displayName: Dataverse Exporter Log Level + path: logging.dataverseExporterLogLevel + - description: 'Log level for the lightspeed-service-api container. Supports + standard Python log levels: DEBUG, INFO, WARNING, ERROR, CRITICAL.' + displayName: Lightspeed Stack Log Level + path: logging.lightspeedStackLogLevel + - description: Log level configuration for the OGX/llama-stack container. Supports + standard levels (INFO, DEBUG) or fine-grained control using format "component=level,component=level" + (e.g., "core=debug,providers=info"). Defaults to "all=info" if empty. + displayName: OGX Log Level + path: logging.ogxLogLevel - description: Name of the model to use at the API endpoint provided in LLMEndpoint displayName: Model Name path: modelName diff --git a/internal/controller/lcore_deployment.go b/internal/controller/lcore_deployment.go index 5f0c43a5..d5b7773f 100644 --- a/internal/controller/lcore_deployment.go +++ b/internal/controller/lcore_deployment.go @@ -20,7 +20,9 @@ import ( "context" "fmt" "path" + "slices" "strconv" + "strings" common_helper "github.com/openstack-k8s-operators/lib-common/modules/common/helper" apiv1beta1 "github.com/openstack-lightspeed/operator/api/v1beta1" @@ -59,7 +61,7 @@ func buildLCorePodTemplateSpec(h *common_helper.Helper, ctx context.Context, ins if err != nil { return corev1.PodTemplateSpec{}, fmt.Errorf("failed to build llama-stack env vars: %w", err) } - lsEnvVars := buildLightspeedStackEnvVars() + lsEnvVars := buildLightspeedStackEnvVars(instance) // Llama Stack container mounts: its config + shared + cache + vector_store_db data llamaStackMounts := []corev1.VolumeMount{} @@ -131,7 +133,7 @@ func buildLCorePodTemplateSpec(h *common_helper.Helper, ctx context.Context, ins Args: []string{ "--mode", "openshift", "--config", path.Join(ExporterConfigMountPath, ExporterConfigFilename), - "--log-level", "INFO", + "--log-level", instance.Spec.Logging.DataverseExporterLogLevel, "--data-dir", LCoreUserDataMountPath, }, VolumeMounts: []corev1.VolumeMount{ @@ -586,10 +588,15 @@ func buildLlamaStackEnvVars(h *common_helper.Helper, ctx context.Context, instan // Postgres password for ${env.POSTGRES_PASSWORD} substitution in llama-stack config envVars = append(envVars, buildPostgresPasswordEnvVar()) - // Logging configuration + // Logging configuration - set both for compatibility with llama-stack and OGX + ogxLogLevel := getOGXLogLevel(instance) envVars = append(envVars, corev1.EnvVar{ Name: "LLAMA_STACK_LOGGING", - Value: "all=info", + Value: ogxLogLevel, + }) + envVars = append(envVars, corev1.EnvVar{ + Name: "OGX_LOGGING", + Value: ogxLogLevel, }) envVars = append(envVars, corev1.EnvVar{ @@ -619,11 +626,11 @@ func buildPostgresPasswordEnvVar() corev1.EnvVar { } // buildLightspeedStackEnvVars builds environment variables for the lightspeed-stack container. -func buildLightspeedStackEnvVars() []corev1.EnvVar { +func buildLightspeedStackEnvVars(instance *apiv1beta1.OpenStackLightspeed) []corev1.EnvVar { return []corev1.EnvVar{ { - Name: "LOG_LEVEL", - Value: "INFO", + Name: "LIGHTSPEED_STACK_LOG_LEVEL", + Value: instance.Spec.Logging.LightspeedStackLogLevel, }, buildPostgresPasswordEnvVar(), } @@ -659,6 +666,28 @@ func buildLightspeedStackReadinessProbe() *corev1.Probe { } } +// getOGXLogLevel returns the log level for OGX/llama-stack container. +// Supports either standard levels (INFO, DEBUG, WARNING, ERROR, CRITICAL) or fine-grained control. +// Examples: "INFO" -> "all=info", "DEBUG" -> "all=debug", "core=debug,providers=info" -> "core=debug,providers=info" +// Defaults to "all=info" if not specified. +func getOGXLogLevel(instance *apiv1beta1.OpenStackLightspeed) string { + logLevel := instance.Spec.Logging.OGXLogLevel + if logLevel == "" { + return "all=info" + } + + // If it's a simple level (INFO, DEBUG, etc.), convert to "all=" format + // Otherwise, pass through for fine-grained control (e.g., "core=debug,providers=info") + upperLogLevel := strings.ToUpper(logLevel) + allowedLogLevels := []string{"DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"} + if slices.Contains(allowedLogLevels, upperLogLevel) { + return fmt.Sprintf("all=%s", strings.ToLower(logLevel)) + } + + // Pass through as-is for fine-grained control + return logLevel +} + // buildConfigMapAnnotations builds annotations with configmap resource versions // so that changes to the configmaps trigger a deployment rollout. func buildConfigMapAnnotations(h *common_helper.Helper, ctx context.Context) (map[string]string, error) { diff --git a/test/kuttl/common/openstack-lightspeed-instance/assert-openstack-lightspeed-instance.yaml b/test/kuttl/common/openstack-lightspeed-instance/assert-openstack-lightspeed-instance.yaml index cf290cf6..4356761a 100644 --- a/test/kuttl/common/openstack-lightspeed-instance/assert-openstack-lightspeed-instance.yaml +++ b/test/kuttl/common/openstack-lightspeed-instance/assert-openstack-lightspeed-instance.yaml @@ -149,9 +149,47 @@ spec: template: spec: containers: - - name: llama-stack - - name: lightspeed-service-api - - name: lightspeed-to-dataverse-exporter + - name: llama-stack + env: + - name: OPENSTACK_LIGHTSPEED_PROVIDER_API_KEY + valueFrom: + secretKeyRef: + key: apitoken + name: openstack-lightspeed-apitoken + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + key: password + name: lightspeed-postgres-secret + - name: LLAMA_STACK_LOGGING + value: all=debug + - name: OGX_LOGGING + value: all=debug + - name: VECTOR_DB_DATA_PATH + value: /vector-db-discovered-values + - name: REQUESTS_CA_BUNDLE + value: /etc/certs/additional-ca/cert.crt + - name: SSL_CERT_FILE + value: /etc/certs/additional-ca/cert.crt + - name: lightspeed-service-api + env: + - name: LIGHTSPEED_STACK_LOG_LEVEL + value: WARNING + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + key: password + name: lightspeed-postgres-secret + - name: lightspeed-to-dataverse-exporter + args: + - --mode + - openshift + - --config + - /etc/config/config.yaml + - --log-level + - DEBUG + - --data-dir + - /tmp/data status: replicas: 1 readyReplicas: 1 @@ -241,6 +279,9 @@ spec: llmProjectID: test-project-id llmDeploymentName: test-deployment-name llmAPIVersion: v1 + logging: + ogxLogLevel: DEBUG + lightspeedStackLogLevel: WARNING status: conditions: - type: Ready diff --git a/test/kuttl/common/openstack-lightspeed-instance/create-openstack-lightspeed-instance.yaml b/test/kuttl/common/openstack-lightspeed-instance/create-openstack-lightspeed-instance.yaml index c1961d10..41b4ebcc 100644 --- a/test/kuttl/common/openstack-lightspeed-instance/create-openstack-lightspeed-instance.yaml +++ b/test/kuttl/common/openstack-lightspeed-instance/create-openstack-lightspeed-instance.yaml @@ -14,3 +14,7 @@ spec: llmDeploymentName: test-deployment-name llmAPIVersion: v1 enableOCPRAG: false + logging: + ogxLogLevel: DEBUG + lightspeedStackLogLevel: WARNING + dataverseExporterLogLevel: DEBUG diff --git a/test/kuttl/tests/update-openstacklightspeed/07-update-openstack-lightspeed-instance.yaml b/test/kuttl/tests/update-openstacklightspeed/07-update-openstack-lightspeed-instance.yaml index 6d5736c5..a612acbb 100644 --- a/test/kuttl/tests/update-openstacklightspeed/07-update-openstack-lightspeed-instance.yaml +++ b/test/kuttl/tests/update-openstacklightspeed/07-update-openstack-lightspeed-instance.yaml @@ -35,3 +35,7 @@ spec: # We can set it to `true` once the OCP contant is present in the image. enableOCPRAG: false ocpVersionOverride: "4.16" + logging: + ogxLogLevel: "core=debug,providers=info" + lightspeedStackLogLevel: ERROR + dataverseExporterLogLevel: ERROR diff --git a/test/kuttl/tests/update-openstacklightspeed/08-assert-openstacklightspeed-update.yaml b/test/kuttl/tests/update-openstacklightspeed/08-assert-openstacklightspeed-update.yaml index f98804db..1a6add7e 100644 --- a/test/kuttl/tests/update-openstacklightspeed/08-assert-openstacklightspeed-update.yaml +++ b/test/kuttl/tests/update-openstacklightspeed/08-assert-openstacklightspeed-update.yaml @@ -70,8 +70,39 @@ spec: template: spec: containers: - - name: llama-stack - - name: lightspeed-service-api + - name: llama-stack + env: + - name: OPENSTACK_LIGHTSPEED_PROVIDER_API_KEY + valueFrom: + secretKeyRef: + key: apitoken + name: openstack-lightspeed-apitoken-update + - name: VLLM_URL + value: http://mock-llm-api-server-pod-UPDATE:8000/v1 + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + key: password + name: lightspeed-postgres-secret + - name: LLAMA_STACK_LOGGING + value: core=debug,providers=info + - name: OGX_LOGGING + value: core=debug,providers=info + - name: VECTOR_DB_DATA_PATH + value: /vector-db-discovered-values + - name: REQUESTS_CA_BUNDLE + value: /etc/certs/additional-ca/cert.crt + - name: SSL_CERT_FILE + value: /etc/certs/additional-ca/cert.crt + - name: lightspeed-service-api + env: + - name: LIGHTSPEED_STACK_LOG_LEVEL + value: ERROR + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + key: password + name: lightspeed-postgres-secret status: replicas: 1 readyReplicas: 1 @@ -157,6 +188,9 @@ spec: transcriptsDisabled: true enableOCPRAG: false ocpVersionOverride: "4.16" + logging: + ogxLogLevel: "core=debug,providers=info" + lightspeedStackLogLevel: ERROR status: conditions: - type: Ready