lightrun-platform · yitzhaktal · May 13, 2025 · May 7, 2025
diff --git a/chart/templates/backend-deployment.yaml b/chart/templates/backend-deployment.yaml
@@ -60,6 +60,13 @@ spec:
       {{- end }}
       serviceAccountName: {{ template "lightrun-be.serviceAccountName" . }}
       volumes:
+        - name: encryption-keys
+          secret: 
+            secretName: {{ include "secrets.backend.name" . }}
+            optional: false
+            items: 
+              # Only select items that start with encryption-key-
+              {{- include "encryption.key.items" . | nindent 18 }}
       {{- include "lightrun-backend.volumes.asyncProfiler" . | nindent 8 }}
       {{- if and .Values.general.internal_tls.enabled .Values.general.internal_tls.certificates.existing_ca_secret_name  }}  
         - name: ca-cert
@@ -123,6 +130,9 @@ spec:
             "/usr/src/lightrun/{{ .Values.deployments.backend.jar_name }}"
           ]
           volumeMounts:
+            - name: encryption-keys
+              mountPath: /encryption-keys
+              readOnly: true
           {{- include "lightrun-backend.volumeMounts.asyncProfiler" . | nindent 12 }}
             - name: jcache-config
               mountPath: "/jcache-config"
@@ -183,6 +193,8 @@ spec:
             - secretRef:
                 name: {{ include "secrets.backend.name" . }}
           env:
+            - name: SERVER_SECURITY_ENCRYPTION-KEYS-PATH
+              value: file:/encryption-keys
             - name: LIGHTRUN_HOSTNAME
               value: {{ .Values.general.name }}
             - name: POD_NAME

diff --git a/chart/templates/helpers/_helpers.tpl b/chart/templates/helpers/_helpers.tpl
@@ -732,3 +732,75 @@ Returns:
 {{- end }}
 {{- $matching_queue }}
 {{- end }}
+
+{{/*
+Get encryption key for Lightrun deployment. Handles existing keys, random generation, or user-provided values.
+*/}}
+{{- define "secrets.encryption-key" -}}
+    {{- if .Values.general.deploy_secrets.enabled }}
+        {{- if (not .Values.secrets.keysEncryption.userEncryptionKey) }}
+            {{- $secretObj := (lookup "v1" "Secret" .Release.Namespace (include "secrets.backend.name" .)) }}
+            {{/* Case 1: take existing generated key from secret */}}
+            {{- if and $secretObj (hasKey $secretObj.data ( include "secrets.encryption-key-name" . ) ) }}
+                {{- index $secretObj.data ( include "secrets.encryption-key-name" . ) | b64dec }}
+            {{/* Case 2: generate random encryption key */}}
+            {{- else }}
+                {{- randBytes 32 }}
+            {{- end }}
+        {{/* Case 3: take user provided encryption key from values */}}
+        {{- else }}
+            {{- .Values.secrets.keysEncryption.userEncryptionKey }}
+        {{- end }}
+    {{- end }}
+{{- end }}
+
+{{- define "secrets.encryption-key-name" -}}
+    {{- $rotateKey := .Values.secrets.keysEncryption.rotateKey }}
+    {{- $defaultKey := "encryption-key-0" }}
+    {{- $keyPrefix := "encryption-key-" }}
+    {{- $secret := (lookup "v1" "Secret" .Release.Namespace (include "secrets.backend.name" .)) }}
+
+    {{- $maxIndex := -1 }}
+    {{- if $secret }}
+        {{- range $k, $ := $secret.data }}
+            {{- if hasPrefix $keyPrefix $k }}
+                {{- $suffix := trimPrefix $keyPrefix $k }}
+                {{- $num := int $suffix }}
+                {{- if gt $num $maxIndex }}
+                    {{- $maxIndex = $num }}
+                {{- end }}
+            {{- end }}
+        {{- end }}
+    {{- end }}
+
+    {{- if eq $maxIndex -1 }}
+        {{- $defaultKey }}
+    {{- else if $rotateKey }}
+        {{- printf "encryption-key-%d" (add1 $maxIndex) }}
+    {{- else }}
+        {{- printf "encryption-key-%d" $maxIndex }}
+    {{- end }}
+{{- end }}
+
+{{/* Helper function to render encryption key items */}}
+{{- define "encryption.key.items" -}}
+{{- $secret := (lookup "v1" "Secret" .Release.Namespace (include "secrets.backend.name" .)) -}}
+{{- $hasEncryptionKeys := false -}}
+{{- if $secret -}}
+{{ range $key, $value := $secret.data }}
+{{ if hasPrefix "encryption-key-" $key }}
+{{- $hasEncryptionKeys = true }}
+- key: {{ $key }}
+  path: {{ $key }}
+{{- end }}
+{{- end }}
+{{- end }}
+{{ if not $hasEncryptionKeys }}
+- key: encryption-key-0
+  path: encryption-key-0
+{{- end }}
+{{ if .Values.secrets.keysEncryption.rotateKey }}
+- key: {{ include "secrets.encryption-key-name" . }}
+  path: {{ include "secrets.encryption-key-name" . }}
+{{- end }}
+{{- end }}
diff --git a/chart/templates/secrets.yaml b/chart/templates/secrets.yaml
@@ -40,6 +40,7 @@ stringData:
   LICENSE_SIGNATURE: {{ .Values.secrets.license.signature | quote }}
   SPRING_RABBITMQ_USERNAME: {{ .Values.secrets.mq.user | default "" | quote }}
   SPRING_RABBITMQ_PASSWORD: {{ .Values.secrets.mq.password | default "" | quote }}
+  {{ include "secrets.encryption-key-name" . }}: {{include "secrets.encryption-key" . }}
 
   # Optional fields
   {{- if .Values.secrets.defaults.datadog_api_key }}

diff --git a/chart/values.yaml b/chart/values.yaml
@@ -508,6 +508,25 @@ secrets:
       # 3. For private registry. If dockerhub_config is set to `null`, chart will not use any imagePullSecrets.
       existingSecret: ""
       configContent: ""
+  keysEncryption:
+    # The encryption key configuration for the backend service.
+    # PREFERRED APPROACH: External Secret
+    # - Create a Kubernetes secret containing the encryption key outside of this chart, see:
+    #   https://github.com/lightrun-platform/lightrun-helm-chart/blob/main/docs/installation/secrets.md
+    # - This is the recommended approach for all environments, especially production
+    #
+    # ALTERNATIVE APPROACH: Chart-managed Secret
+    # - Let the chart create and manage the secret (requires deploy_secrets.enabled=true)
+    # - Not recommended for production environments
+    #
+    # IMPORTANT SECURITY NOTES:
+    # - While you can provide an encryption key via userEncryptionKey, storing keys in values.yaml
+    #   is not secure for production environments. Use external secrets management solution
+    # - For GitOps environments, you MUST provide a stable encryption key externally
+    #
+    # For more details, see: https://github.com/lightrun-platform/lightrun-helm-chart/blob/main/docs/advanced/encryption_keys.md
+    userEncryptionKey: ""
+    rotateKey: false
 
 ################
 ## Deployments

diff --git a/docs/advanced/encryption_keys.md b/docs/advanced/encryption_keys.md
@@ -0,0 +1,83 @@
+# Encryption Keys
+
+This document describes how to manage encryption keys for the Lightrun backend service. These keys are used to encrypt sensitive data such as:
+- API keys for integrations (Datadog, SIEM Integration using Splunk, etc.)
+- OpenAI Keys
+- Other sensitive configuration data
+
+You can either:
+
+- **Recommended**: Connect to an External Encryption Key (`general.deploy_secrets.enabled: false`)
+- **Not recommended for production**: Deploy a Chart-managed Encryption Key (`general.deploy_secrets.enabled: true`)
+
+## Recommended Approach: External Encryption-Key (`general.deploy_secrets.enabled: false`)
+
+For production environments, it's strongly recommended to manage encryption keys externally rather than letting the chart create and manage them. This approach provides better security and control over the encryption keys.
+
+### Creating an External Encryption Key
+
+1. Generate a secure encryption key:
+   ```bash
+   openssl rand -base64 32
+   ```
+
+   > [!IMPORTANT]
+   > The encryption key must be:
+   > - Exactly 32 byte
+   > - Unique for each environment
+   > - Stored securely (e.g., in a password manager or secret management system)
+
+2. Create a Kubernetes secret containing all required fields, including the encryption key. See [Secrets Documentation](../installation/secrets.md) for the complete list of required fields.
+
+3. Configure the chart to use the external secret:
+```yaml
+general:
+  deploy_secrets:
+    enabled: false
+    existing_secrets:
+      backend: <secret-name>
+```
+
+### Key Rotation with External Secrets
+
+When rotating encryption keys, ensure your secret contains both the old and new encryption keys in the following format:
+- `encryption-key-0`: The original key
+- `encryption-key-1`: The new key
+
+> [!IMPORTANT]
+> - If you have created your own encryption key, store it in a Kubernetes secret and reference it via `existing_secrets.backend`. Do not put the key directly in values.yaml
+> - For GitOps environments, you MUST provide stable encryption keys externally
+> - Keep a secure backup of your encryption keys
+> - Follow your organization's key rotation policies
+
+## Alternative Approach: Chart-managed Encryption-Key (`general.deploy_secrets.enabled: true`)
+
+For testing, POC, or development environments, you can let the chart create and manage the encryption key either by providing your own key via `keysEncryption.userEncryptionKey` or letting the chart auto-generate one if this parameter is empty. This is not recommended for production use.
+
+### Steps to Use Chart-managed Secret
+
+```yaml
+secrets:
+  keysEncryption:
+    userEncryptionKey: ""  # Leave empty to auto-generate
+    rotateKey: false       # Set to true to enable key rotation
+```
+
+> [!WARNING]
+> This approach is not recommended for production environments. Use external secrets instead.> 
+### Key Rotation with Chart-managed secret
+
+When `rotateKey` is set to `true`, the chart will:
+1. Generate a new key
+2. Store both old and new keys in the secret
+3. The backend will use both keys for decryption but only the new key for encryption
+
+### Limitations of Chart-managed Keys
+
+- For GitOps users (ArgoCD, Flux), each GitOps sync generates a new random key, making it incompatible with existing encrypted data.
+- While the chart provides basic key rotation, there is no backup capability and no automated rotation policies (rotation can be done proactivly in a manual way).
+
+
+> [!WARNING]
+> For GitOps or production environments, you MUST use external secrets management to maintain a stable encryption key.
+
diff --git a/docs/installation/secrets.md b/docs/installation/secrets.md
@@ -17,8 +17,28 @@ general:
       keycloak: ""
 ```
 - _(This is relevant only when `deploy_secrets: false`.)_
+
+### **Mandatory Secret Fields**
+
+When managing secrets externally, ensure the following fields are present in your secret (See the [secrets template](https://github.com/lightrun-platform/lightrun-helm-chart/blob/main/chart/templates/secrets.yaml#L31) for reference):
+
+| Environment Variable | Description | Value Source |
+|---------------------|-------------|--------------|
+| `SPRING_SECURITY_KEYCLOAK_CLI_PASSWORD` | Keycloak admin password | `secrets.keycloak.password` |
+| `SPRING_MAIL_PASSWORD` | Mail server password | `secrets.defaults.mail_password` |
+| `SPRING_FLYWAY_PASSWORD` | DB password | `secrets.db.password` |
+| `SPRING_FLYWAY_USER` | DB user | `secrets.db.user` |
+| `SPRING_DATASOURCE_USERNAME` | DB username | `secrets.db.user` |
+| `SPRING_DATASOURCE_PASSWORD` | DB password | `secrets.db.password` |
+| `KEYSTORE_PASSWORD` | Java Keystore password | `secrets.defaults.keystore_password` |
+| `LICENSE_CONTENT` | Lightrun license content | `secrets.license.content` |
+| `LICENSE_SIGNATURE` | Lightrun license signature | `secrets.license.signature` |
+| `SPRING_RABBITMQ_USERNAME` | RabbitMQ username | `secrets.mq.user` |
+| `SPRING_RABBITMQ_PASSWORD` | RabbitMQ password | `secrets.mq.password` |
+| `encryption-key-0` | Backend encryption key (default) | `secrets.keysEncryption.userEncryptionKey` |
+
 > [!WARNING]
-> If managing secrets externally, ensure all required fields are present. See the [secrets template](https://github.com/lightrun-platform/lightrun-helm-chart/blob/main/chart/templates/secrets.yaml#L31) for reference.
+> For encryption keys, it's strongly recommended to provide them as external secrets rather than letting the chart manage them. See [Encryption Keys Documentation](../advanced/encryption_keys.md) for details.
 
 ## **Secrets Configuration**
 ### **Authentication and Access Secrets**