From c986a28721ba2d58e5a69ecd06dd811a556eaa0e Mon Sep 17 00:00:00 2001 From: sylvain senechal Date: Tue, 28 Apr 2026 16:01:29 +0200 Subject: [PATCH 1/2] added kmip encryption tests Issue: ZENKO-5241 --- .github/scripts/end2end/configure-e2e-ctst.sh | 5 + .github/scripts/mocks/setup-kmip.sh | 125 ++++++++++++++++++ tests/ctst/common/hooks.ts | 63 ++++++++- .../features/serverSideEncryption.feature | 77 +++++++++++ tests/ctst/steps/serverSideEncryption.ts | 76 +++++++---- 5 files changed, 317 insertions(+), 29 deletions(-) create mode 100755 .github/scripts/mocks/setup-kmip.sh diff --git a/.github/scripts/end2end/configure-e2e-ctst.sh b/.github/scripts/end2end/configure-e2e-ctst.sh index 5acaa1331c..d3f7e0d033 100755 --- a/.github/scripts/end2end/configure-e2e-ctst.sh +++ b/.github/scripts/end2end/configure-e2e-ctst.sh @@ -106,3 +106,8 @@ kubectl run kafka-topics \ kafka-topics.sh --create --topic $AZURE_ARCHIVE_STATUS_TOPIC_2_NV --partitions 10 --bootstrap-server $KAFKA_HOST_PORT --if-not-exists ; \ kafka-topics.sh --create --topic $AZURE_ARCHIVE_STATUS_TOPIC_2_V --partitions 10 --bootstrap-server $KAFKA_HOST_PORT --if-not-exists ; \ kafka-topics.sh --create --topic $AZURE_ARCHIVE_STATUS_TOPIC_2_S --partitions 10 --bootstrap-server $KAFKA_HOST_PORT --if-not-exists" + +# KMIP mock setup +# Deploy PyKMIP server (infra only, does NOT patch the CR). +# The CR is patched later, after file-backend SSE tests have run. +bash "$(dirname "$0")/../mocks/setup-kmip.sh" diff --git a/.github/scripts/mocks/setup-kmip.sh b/.github/scripts/mocks/setup-kmip.sh new file mode 100755 index 0000000000..51878ca4e7 --- /dev/null +++ b/.github/scripts/mocks/setup-kmip.sh @@ -0,0 +1,125 @@ +#!/bin/bash +# setup-kmip.sh — Deploy PyKMIP mock server for KMIP SSE testing. +# Idempotent +# +# Deploys PyKMIP infra (certs, pod, service). The Zenko CR is patched +# by the CTST Before hook when @ServerSideEncryptionKmip tests start. + +set -euo pipefail + +ZENKO_NAME="${ZENKO_NAME:-end2end}" +NAMESPACE="${NAMESPACE:-default}" + +if kubectl get deployment pykmip -n "${NAMESPACE}" &>/dev/null; then + echo "PyKMIP already deployed, skipping" + exit 0 +fi + +# 1. Certs + secrets + +if kubectl get secret "${ZENKO_NAME}-kmip-certs" -n "${NAMESPACE}" &>/dev/null; then + echo "KMIP secrets already exist, skipping cert generation" +else + echo "Generating KMIP TLS certificates..." + D=$(mktemp -d) + trap 'rm -rf "$D"' EXIT + + openssl genrsa -out "$D/ca.key" 4096 2>/dev/null + openssl req -new -x509 -key "$D/ca.key" -out "$D/ca.pem" \ + -days 3650 -subj "/CN=KMIP-CA" 2>/dev/null + + openssl genrsa -out "$D/server.key" 4096 2>/dev/null + openssl req -new -key "$D/server.key" -out "$D/server.csr" \ + -subj "/CN=pykmip" 2>/dev/null + openssl x509 -req -in "$D/server.csr" -CA "$D/ca.pem" -CAkey "$D/ca.key" \ + -CAcreateserial -out "$D/server.crt" -days 3650 \ + -extfile <(printf "subjectAltName=DNS:pykmip,DNS:pykmip.%s.svc.cluster.local" "$NAMESPACE") \ + 2>/dev/null + + openssl genrsa -out "$D/client.key" 4096 2>/dev/null + openssl req -new -key "$D/client.key" -out "$D/client.csr" \ + -subj "/CN=cloudserver-client" 2>/dev/null + openssl x509 -req -in "$D/client.csr" -CA "$D/ca.pem" -CAkey "$D/ca.key" \ + -CAcreateserial -out "$D/client.crt" -days 3650 \ + -extfile <(printf "extendedKeyUsage=clientAuth") 2>/dev/null + + kubectl create secret generic "${ZENKO_NAME}-kmip-certs" \ + --from-file=ca.pem="$D/ca.pem" --from-file=cert.pem="$D/client.crt" \ + --from-file=key.pem="$D/client.key" \ + -n "${NAMESPACE}" --dry-run=client -o yaml | kubectl apply -f - + + kubectl create secret generic pykmip-server-certs \ + --from-file=ca.crt="$D/ca.pem" --from-file=server.crt="$D/server.crt" \ + --from-file=server.key="$D/server.key" \ + -n "${NAMESPACE}" --dry-run=client -o yaml | kubectl apply -f - +fi + +# 2. PyKMIP startup script + +kubectl create configmap pykmip-server-script -n "${NAMESPACE}" --dry-run=client -o yaml \ + --from-literal=run_pykmip.py=' +import logging; from kmip.services.server import KmipServer +logging.basicConfig(level=logging.INFO) +server = KmipServer(hostname="0.0.0.0", port=5696, + certificate_path="/certs/server.crt", key_path="/certs/server.key", + ca_path="/certs/ca.crt", auth_suite="TLS1.2", config_path=None, + enable_tls_client_auth=True, database_path="/tmp/pykmip.db") +with server: server.serve() +' | kubectl apply -f - + +# 3. Deploy PyKMIP pod + service (inline YAML) + +if ! kubectl get deployment pykmip -n "${NAMESPACE}" &>/dev/null; then + kubectl apply -n "${NAMESPACE}" -f - <<'YAML' +apiVersion: v1 +kind: Service +metadata: + name: pykmip +spec: + selector: { name: pykmip } + ports: [{ name: kmip, port: 5696, targetPort: 5696 }] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: pykmip + labels: { name: pykmip } +spec: + replicas: 1 + selector: + matchLabels: { name: pykmip } + template: + metadata: + labels: { name: pykmip } + spec: + initContainers: + - name: install + image: docker.io/library/python:3.10-slim + command: [pip, install, --target=/pykmip-libs, pykmip==0.10.0, -q] + volumeMounts: [{ name: pykmip-libs, mountPath: /pykmip-libs }] + containers: + - name: pykmip + image: docker.io/library/python:3.10-slim + command: [python3, /scripts/run_pykmip.py] + env: [{ name: PYTHONPATH, value: /pykmip-libs }] + ports: [{ containerPort: 5696 }] + readinessProbe: + tcpSocket: { port: 5696 } + initialDelaySeconds: 5 + periodSeconds: 3 + volumeMounts: + - { name: certs, mountPath: /certs, readOnly: true } + - { name: scripts, mountPath: /scripts, readOnly: true } + - { name: pykmip-libs, mountPath: /pykmip-libs } + volumes: + - { name: certs, secret: { secretName: pykmip-server-certs } } + - { name: scripts, configMap: { name: pykmip-server-script } } + - { name: pykmip-libs, emptyDir: {} } +YAML + echo "Waiting for PyKMIP..." + kubectl wait --for=condition=Available deployment/pykmip -n "${NAMESPACE}" --timeout=5m +else + echo "PyKMIP already deployed" +fi + +echo "PyKMIP infra ready" diff --git a/tests/ctst/common/hooks.ts b/tests/ctst/common/hooks.ts index d06982142d..da7e1a157f 100644 --- a/tests/ctst/common/hooks.ts +++ b/tests/ctst/common/hooks.ts @@ -6,7 +6,7 @@ import { ITestCaseHookParameter, } from '@cucumber/cucumber'; import Zenko from '../world/Zenko'; -import { CacheHelper, Identity } from 'cli-testing'; +import { CacheHelper, Identity, WorkCoordination } from 'cli-testing'; import { prepareQuotaScenarios, teardownQuotaScenarios } from 'steps/quotas/quotas'; import { prepareUtilizationScenarios } from 'steps/utilization/utilizationAPI'; import { prepareMetricsScenarios } from './utils'; @@ -16,6 +16,7 @@ import { displayDebuggingInformation, preparePRA } from 'steps/pra'; import { cleanupAccount, } from './utils'; +import { createKubeCustomObjectClient, waitForZenkoToStabilize } from 'steps/utils/kubernetes'; import 'cli-testing/hooks/KeycloakSetup'; import 'cli-testing/hooks/Logger'; @@ -33,6 +34,7 @@ const noParallelRun = atMostOnePicklePerTag([ '@AfterAll', '@PRA', '@ColdStorage', + '@ServerSideEncryption', ...replicationLockTags ]); @@ -75,6 +77,65 @@ Before({ tags: '@PrepareStorageUsageReportingScenarios', timeout: 1200000 }, asy }); }); +Before({ tags: '@ServerSideEncryptionKmip', timeout: 15 * 60 * 1000 }, + // Patch the Zenko CR with KMIP configuration before running any KMIP-related tests + async function (this: Zenko) { + const lockName = `kmip-cr-patch-${process.ppid}`; + await WorkCoordination.runOnceAcrossWorkers( + { lockName, logger: this.logger }, + async () => { + const namespace = 'default'; + const zenkoName = 'end2end'; + const client = createKubeCustomObjectClient(this); + const cr = await client.getNamespacedCustomObject({ + group: 'zenko.io', + version: 'v1alpha2', + namespace, + plural: 'zenkos', + name: zenkoName, + }) as { + spec?: { + kms?: { + kmip?: { + providerName?: string; + tlsAuth?: { tlsSecretName?: string }; + endpoints?: { host?: string; port?: number }[]; + }; + }; + }; + }; + const alreadyConfigured = + cr?.spec?.kms?.kmip?.providerName === 'pykmip' + && cr?.spec?.kms?.kmip?.endpoints?.some( + ep => ep.host === `pykmip.${namespace}.svc.cluster.local` && ep.port === 5696, + ); + if (alreadyConfigured) { + return; + } + + const kmipValue = { + providerName: 'pykmip', + tlsAuth: { tlsSecretName: `${zenkoName}-kmip-certs` }, + endpoints: [{ + host: `pykmip.${namespace}.svc.cluster.local`, + port: 5696, + }], + }; + + await client.patchNamespacedCustomObject({ + group: 'zenko.io', + version: 'v1alpha2', + namespace, + plural: 'zenkos', + name: zenkoName, + body: [{ op: 'add', path: '/spec/kms', value: { kmip: kmipValue } }], + }); + await waitForZenkoToStabilize(this, true); + }, + ); + }, +); + After(async function (this: Zenko, results) { // Reset any configuration set on the endpoint (ssl, port) CacheHelper.parameters.ssl = this.parameters.ssl; diff --git a/tests/ctst/features/serverSideEncryption.feature b/tests/ctst/features/serverSideEncryption.feature index 95f07a25e5..19ddc27d9a 100644 --- a/tests/ctst/features/serverSideEncryption.feature +++ b/tests/ctst/features/serverSideEncryption.feature @@ -85,3 +85,80 @@ Feature: Server Side Encryption Given a "Non versioned" bucket When the user gets bucket encryption Then it should fail with error "ServerSideEncryptionConfigurationNotFoundError" + + # KMIP backend tests + # These scenarios require a PyKMIP server to be deployed and Zenko to be + # reconfigured with spec.kms.kmip before running. The previous @ServerSideEncryptionFileBackend + # tests will not work once KMIP is configured on the ZENKO custom resource. + + @2.14.0 + @PreMerge + @ServerSideEncryption + @ServerSideEncryptionKmip + Scenario Outline: KMIP: should encrypt object when bucket encryption is and object encryption is + Given a "Non versioned" bucket + And bucket encryption is set to "" with key "" + Then the bucket encryption is verified for algorithm "" and key "" + When an object "" is uploaded with SSE algorithm "" and key "" + Then the PutObject response should have SSE algorithm "" and KMS key "" + Then the GetObject should return the uploaded body with SSE algorithm "" and KMS key "" + + Examples: No bucket encryption + | objectName | bucketAlgo | bucketKeyId | objectAlgo | objectKeyId | expectedAlgo | expectedKeyId | + | kmip-none-none | | | | | | absent | + | kmip-none-aes | | | AES256 | | AES256 | absent | + + Examples: No bucket encryption, aws:kms + | objectName | bucketAlgo | bucketKeyId | objectAlgo | objectKeyId | expectedAlgo | expectedKeyId | + | kmip-none-kms | | | aws:kms | | aws:kms | generated | + + Examples: Bucket AES256 + | objectName | bucketAlgo | bucketKeyId | objectAlgo | objectKeyId | expectedAlgo | expectedKeyId | + | kmip-aes-none | AES256 | | | | AES256 | absent | + | kmip-aes-aes | AES256 | | AES256 | | AES256 | absent | + | kmip-aes-kms | AES256 | | aws:kms | | aws:kms | generated | + + Examples: Bucket aws:kms (default key) + | objectName | bucketAlgo | bucketKeyId | objectAlgo | objectKeyId | expectedAlgo | expectedKeyId | + | kmip-kms-none | aws:kms | | | | aws:kms | generated | + | kmip-kms-aes | aws:kms | | AES256 | | AES256 | absent | + | kmip-kms-kms | aws:kms | | aws:kms | | aws:kms | generated | + + @2.14.0 + @PreMerge + @ServerSideEncryption + @ServerSideEncryptionKmip + Scenario: KMIP: DeleteBucketEncryption removes default encryption + Given a "Non versioned" bucket + And bucket encryption is set to "AES256" with key "" + When an object "kmip-enc-obj" is uploaded with SSE algorithm "" and key "" + Then the GetObject should return the uploaded body with SSE algorithm "AES256" and KMS key "absent" + When the user deletes bucket encryption + Then the GetObject should return the uploaded body with SSE algorithm "AES256" and KMS key "absent" + When an object "kmip-plain-obj" is uploaded with SSE algorithm "" and key "" + Then the GetObject should return the uploaded body with SSE algorithm "" and KMS key "absent" + + @2.14.0 + @PreMerge + @ServerSideEncryption + @ServerSideEncryptionKmip + Scenario Outline: KMIP: PutObject with invalid SSE parameters returns an error: + Given a "Non versioned" bucket + When an object "" is uploaded with SSE algorithm "" and key "" + Then it should fail with error "InvalidArgument" + + Examples: + | objectName | algo | keyId | + | kmip-invalid-algo | INVALID_ALGO | | + | kmip-aes-kms-err | AES256 | some-key | + + @2.14.0 + @PreMerge + @ServerSideEncryption + @ServerSideEncryptionKmip + Scenario: KMIP: objects in same bucket share the same KMIP master key + Given a "Non versioned" bucket + And bucket encryption is set to "aws:kms" with key "" + When an object "kmip-shared-key-obj-a" is uploaded with SSE algorithm "" and key "" + And an object "kmip-shared-key-obj-b" is uploaded with SSE algorithm "" and key "" + Then objects "kmip-shared-key-obj-a" and "kmip-shared-key-obj-b" share the same KMS key diff --git a/tests/ctst/steps/serverSideEncryption.ts b/tests/ctst/steps/serverSideEncryption.ts index 9c91bca7af..ccaea6860e 100644 --- a/tests/ctst/steps/serverSideEncryption.ts +++ b/tests/ctst/steps/serverSideEncryption.ts @@ -1,7 +1,6 @@ import { When, Then } from '@cucumber/cucumber'; -import { CacheHelper, Identity, S3 } from 'cli-testing'; +import { S3 } from 'cli-testing'; import { - S3Client, GetObjectCommand, PutObjectCommand, type ServerSideEncryption, @@ -16,28 +15,6 @@ import assert from 'assert'; // - cli-testing's S3.getObject writes the body to a shared file (out.loc) // which races under parallel execution. // Long term solution : consider dropping cli-testing sdk wrapper : https://scality.atlassian.net/browse/ZENKO-5247 -function buildS3Client(): S3Client { - const credentials = Identity.getCurrentCredentials(); - const ssl = CacheHelper.parameters?.ssl === false - ? 'http://' : 'https://'; - const host = CacheHelper.parameters?.ip - || `s3.${CacheHelper.parameters?.subdomain}`; - const port = CacheHelper.parameters?.port || '80'; - return new S3Client({ - region: 'us-east-1', - endpoint: `${ssl}${host}:${port}`, - forcePathStyle: true, - credentials: { - accessKeyId: credentials.accessKeyId, - secretAccessKey: credentials.secretAccessKey, - ...(credentials.sessionToken - ? { sessionToken: credentials.sessionToken } - : {}), - }, - tls: CacheHelper.parameters?.ssl !== false, - maxAttempts: 1, - }); -} When('bucket encryption is set to {string} with key {string}', async function (this: Zenko, algo: string, keyId: string) { @@ -114,7 +91,7 @@ When('an object {string} is uploaded with SSE algorithm {string} and key {string this.addToSaved('objectName', objectName); this.addToSaved('objectBody', SSE_TEST_BODY); const bucket = this.getSaved('bucketName'); - const client = buildS3Client(); + const client = this.createS3Client(); try { const resp = await client.send(new PutObjectCommand({ Bucket: bucket, @@ -158,6 +135,10 @@ Then('the PutObject response should have SSE algorithm {string} and KMS key {str if (expectedKey === 'absent') { assert.strictEqual(result.sseKmsKeyId, undefined, `PutObject: SSEKMSKeyId should be absent, got "${result.sseKmsKeyId}"`); + } else if (expectedKey === 'generated') { + assert.ok(result.sseKmsKeyId, 'PutObject: SSEKMSKeyId should be present'); + assert.ok(isValidSseKmsKeyId(result.sseKmsKeyId), + `PutObject: expected a generated key (hex or KMIP), got "${result.sseKmsKeyId}"`); } else { assert.ok(result.sseKmsKeyId, 'PutObject: SSEKMSKeyId should be present'); } @@ -169,7 +150,7 @@ Then('the GetObject should return the uploaded body with SSE algorithm {string} const bucket = this.getSaved('bucketName'); const objectName = this.getSaved('objectName'); const expectedBody = this.getSaved('objectBody'); - const client = buildS3Client(); + const client = this.createS3Client(); try { const resp = await client.send( new GetObjectCommand({ Bucket: bucket, Key: objectName }), @@ -189,8 +170,8 @@ Then('the GetObject should return the uploaded body with SSE algorithm {string} `GetObject: SSEKMSKeyId should be absent, got "${resp.SSEKMSKeyId}"`); } else if (expectedKey === 'generated') { assert.ok(resp.SSEKMSKeyId, 'GetObject: SSEKMSKeyId should be present'); - assert.match(resp.SSEKMSKeyId, /^[a-f0-9]{64}$/, - `GetObject: expected a generated hex key, got "${resp.SSEKMSKeyId}"`); + assert.ok(isValidSseKmsKeyId(resp.SSEKMSKeyId), + `GetObject: expected a generated key (hex or KMIP), got "${resp.SSEKMSKeyId}"`); } else { assert.strictEqual(resp.SSEKMSKeyId, expectedKey, `GetObject: expected key "${expectedKey}", got "${resp.SSEKMSKeyId}"`); @@ -208,3 +189,42 @@ Then('it should fail with error {string}', `Expected error "${expectedError}" but got: ${result.err}`); }, ); + +Then('objects {string} and {string} share the same KMS key', + async function (this: Zenko, objA: string, objB: string) { + const bucket = this.getSaved('bucketName'); + const client = this.createS3Client(); + try { + const [respA, respB] = await Promise.all([ + client.send(new GetObjectCommand({ Bucket: bucket, Key: objA })), + client.send(new GetObjectCommand({ Bucket: bucket, Key: objB })), + ]); + const keyA = respA.SSEKMSKeyId; + const keyB = respB.SSEKMSKeyId; + assert.ok(keyA, `Object "${objA}" has no SSEKMSKeyId`); + assert.ok(keyB, `Object "${objB}" has no SSEKMSKeyId`); + assert.strictEqual(keyA, keyB, + `Objects in same bucket should share the same KMIP key; got "${keyA}" vs "${keyB}"`); + } finally { + client.destroy(); + } + }, +); + +/** + * Validates if the provided SSE KMS Key ID matches supported backend formats: + * 1. File Backend: A 64-character hex string. + * 2. KMIP Backend: A numeric string OR a specific Scality KMIP ARN. + * @param sseKmsKeyId - The key ID string to validate + * @returns boolean + */ +function isValidSseKmsKeyId(sseKmsKeyId: string | undefined): boolean { + if (!sseKmsKeyId) { + return false; + } + + const isFileBackendKey = /^[a-f0-9]{64}$/.test(sseKmsKeyId); + const isKmipKey = /^(\d+|arn:scality:kms:external:kmip:[a-z0-9]+:key\/\d+)$/.test(sseKmsKeyId); + + return isFileBackendKey || isKmipKey; +} From ab42f856a87e541e2cd26e3e5b11acabee7ed6fc Mon Sep 17 00:00:00 2001 From: sylvain senechal Date: Tue, 28 Apr 2026 16:01:53 +0200 Subject: [PATCH 2/2] bump zenko operator to 1.8.10 Issue: ZENKO-5241 --- solution/deps.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solution/deps.yaml b/solution/deps.yaml index 06c486b12d..b1096c6294 100644 --- a/solution/deps.yaml +++ b/solution/deps.yaml @@ -130,7 +130,7 @@ vault: zenko-operator: sourceRegistry: ghcr.io/scality image: zenko-operator - tag: v1.8.9 + tag: v1.8.10 envsubst: ZENKO_OPERATOR_TAG zookeeper: sourceRegistry: ghcr.io/adobe/zookeeper-operator