From d80ee72a997e7e2d4d5334ab14e2367f6ed382f7 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Mon, 1 Sep 2025 16:58:41 +1000 Subject: [PATCH 1/6] feat: allow the use of self-signed certificates with trustee Signed-off-by: Chris Butler --- README.md | 2 +- ansible/gen-certificate.yaml | 140 ++++++++++++++++++ ansible/init-data-gzipper.yaml | 18 +++ ansible/initdata-default.toml.tpl | 6 + .../hub/trustee/templates/kbs-config-map.yaml | 5 +- charts/hub/trustee/templates/kbs-route.yaml | 3 +- charts/hub/trustee/templates/kbs.yaml | 4 +- charts/hub/trustee/templates/push-secret.yaml | 27 ++++ .../hub/trustee/templates/tls-cert-eso.yaml | 24 +++ charts/hub/trustee/templates/tls-key-eso.yaml | 24 +++ values-simple.yaml | 8 +- 11 files changed, 252 insertions(+), 9 deletions(-) create mode 100644 ansible/gen-certificate.yaml create mode 100644 charts/hub/trustee/templates/push-secret.yaml create mode 100644 charts/hub/trustee/templates/tls-cert-eso.yaml create mode 100644 charts/hub/trustee/templates/tls-key-eso.yaml diff --git a/README.md b/README.md index 67a67879..02cf461a 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Future work includes: - Only currently is known to work with `azure` as the provider of confidential vms via peer-pods. - Only known to work today with everything on one cluster. The work to expand this is in flight. -- If not using ARO you must either provide your own CA signed certs, or use let's encrypt. +- Below version 3.1, if not using ARO you must either provide your own CA signed certs, or use let's encrypt. - Must be on 4.16.14 or later. ## Major versions diff --git a/ansible/gen-certificate.yaml b/ansible/gen-certificate.yaml new file mode 100644 index 00000000..656993a4 --- /dev/null +++ b/ansible/gen-certificate.yaml @@ -0,0 +1,140 @@ +--- +- name: Generate self-signed TLS cert for KBS and push to Kubernetes Secret + hosts: localhost + connection: local + become: false + gather_facts: false + vars: + kubeconfig: "{{ lookup('env', 'KUBECONFIG') }}" + hub_domain: "{{ global.hubClusterDomain | default('none') | lower}}" + namespace: trustee-operator-system + secret_name: kbs-tls-self-signed + common_name: "kbs-trustee-operator-system.{{ hub_domain }}" + days_valid: 365 + renewal_threshold_days: 10 + need_new_cert: false + pre_tasks: + + - name: Check if TLS secret exists + kubernetes.core.k8s_info: + kubeconfig: "{{ kubeconfig }}" + api_version: v1 + kind: Secret + name: "{{ secret_name }}" + namespace: "imperative" + register: existing_secret + ignore_errors: true + + - name: Set fact that certificate doesn't exist + ansible.builtin.set_fact: + need_new_cert: true + when: existing_secret.resources | length == 0 + + - name: Extract existing certificate if secret exists + ansible.builtin.set_fact: + existing_cert_data: "{{ existing_secret.resources[0].data['tls.crt'] | b64decode }}" + when: existing_secret.resources | length > 0 + + - name: Create temporary file for existing certificate analysis + ansible.builtin.tempfile: + state: file + suffix: .crt + register: temp_cert_file + when: existing_secret.resources | length > 0 + + - name: Write existing certificate to temp file + ansible.builtin.copy: + content: "{{ existing_cert_data }}" + dest: "{{ temp_cert_file.path }}" + when: existing_secret.resources | length > 0 + + - name: Get certificate expiry date + community.crypto.x509_certificate_info: + path: "{{ temp_cert_file.path }}" + register: cert_info + when: existing_secret.resources | length > 0 + + - name: Calculate days until expiry + ansible.builtin.set_fact: + days_until_expiry: "{{ ((cert_info.not_after | to_datetime('%Y%m%d%H%M%SZ')) - ansible_date_time.epoch | int | to_datetime('%s')).days }}" + when: existing_secret.resources | length > 0 + + - name: Set fact to generate new certificate if expiring soon + ansible.builtin.set_fact: + need_new_cert: true + when: + - existing_secret.resources | length > 0 + - days_until_expiry | int <= renewal_threshold_days + + - name: Clean up temporary certificate file + ansible.builtin.file: + path: "{{ temp_cert_file.path }}" + state: absent + when: existing_secret.resources | length > 0 + + - name: Display certificate status + ansible.builtin.debug: + msg: > + Certificate status: + {% if existing_secret.resources | length == 0 %} + No existing certificate found. Will generate new certificate. + {% elif need_new_cert %} + Certificate expires in {{ days_until_expiry }} days (threshold: {{ renewal_threshold_days }} days). Will generate new certificate. + {% else %} + Certificate is valid for {{ days_until_expiry }} more days. Skipping certificate generation. + {% endif %} + + - name: Create temporary directory for cert generation + ansible.builtin.tempfile: + state: directory + prefix: kbs-cert- + register: tmpdir + when: need_new_cert + + tasks: + - name: Generate private key + community.crypto.openssl_privatekey: + path: "{{ tmpdir.path }}/tls.key" + size: 4096 + when: need_new_cert + + - name: Generate CSR + community.crypto.openssl_csr: + path: "{{ tmpdir.path }}/tls.csr" + privatekey_path: "{{ tmpdir.path }}/tls.key" + common_name: "kbs-trustee-operator-system" + subject_alt_name: + - "DNS:{{ common_name }}" + when: need_new_cert + + - name: Generate self-signed certificate + community.crypto.x509_certificate: + path: "{{ tmpdir.path }}/tls.crt" + privatekey_path: "{{ tmpdir.path }}/tls.key" + csr_path: "{{ tmpdir.path }}/tls.csr" + provider: selfsigned + selfsigned_not_after: "+{{ days_valid }}d" + when: need_new_cert + + - name: Create or update TLS secret for KBS + kubernetes.core.k8s: + kubeconfig: "{{ kubeconfig }}" + state: present + definition: + apiVersion: v1 + kind: Secret + metadata: + name: "{{ secret_name }}" + namespace: "imperative" + type: kubernetes.io/tls + stringData: + tls.crt: "{{ lookup('file', tmpdir.path + '/tls.crt') }}" + tls.key: "{{ lookup('file', tmpdir.path + '/tls.key') }}" + when: need_new_cert + + - name: Cleanup temporary directory + ansible.builtin.file: + path: "{{ tmpdir.path }}" + state: absent + when: need_new_cert and tmpdir is defined + diff --git a/ansible/init-data-gzipper.yaml b/ansible/init-data-gzipper.yaml index c0ac9584..714d0490 100644 --- a/ansible/init-data-gzipper.yaml +++ b/ansible/init-data-gzipper.yaml @@ -14,6 +14,24 @@ state: directory suffix: initdata register: tmpdir + - name: Read KBS TLS secret from Kubernetes + kubernetes.core.k8s_info: + kubeconfig: "{{ lookup('env', 'KUBECONFIG') }}" + api_version: v1 + kind: Secret + name: kbs-tls-self-signed + namespace: imperative + register: kbs_secret_result + + - name: Extract and decode certificate from secret + ansible.builtin.set_fact: + trustee_cert: "{{ kbs_secret_result.resources[0].data['tls.crt'] | b64decode }}" + when: kbs_secret_result.resources | length > 0 + + - name: Fail if certificate not found + ansible.builtin.fail: + msg: "KBS TLS certificate not found in secret 'kbs-tls-self-signed' in namespace 'imperative'" + when: kbs_secret_result.resources | length == 0 - name: Define temp file paths ansible.builtin.set_fact: diff --git a/ansible/initdata-default.toml.tpl b/ansible/initdata-default.toml.tpl index 271e3483..df053f35 100644 --- a/ansible/initdata-default.toml.tpl +++ b/ansible/initdata-default.toml.tpl @@ -9,6 +9,9 @@ url = "https://kbs-trustee-operator-system.{{ hub_domain }}" [token_configs.kbs] url = "https://kbs-trustee-operator-system.{{ hub_domain }}" +cert = """ +{{ trustee_cert }} +""" ''' "cdh.toml" = ''' @@ -18,4 +21,7 @@ credentials = [] [kbc] name = "cc_kbc" url = "https://kbs-trustee-operator-system.{{ hub_domain }}" +kbs_cert = """ +{{ trustee_cert }} +""" ''' diff --git a/charts/hub/trustee/templates/kbs-config-map.yaml b/charts/hub/trustee/templates/kbs-config-map.yaml index 13c4219c..5c033bfb 100644 --- a/charts/hub/trustee/templates/kbs-config-map.yaml +++ b/charts/hub/trustee/templates/kbs-config-map.yaml @@ -7,8 +7,9 @@ data: kbs-config.toml: | [http_server] sockets = ["0.0.0.0:8080"] - insecure_http = true - + insecure_http = false + private_key = "/etc/https-key/tls.key" + certificate = "/etc/https-cert/tls.crt" [admin] insecure_api = true auth_public_key = "/etc/auth-secret/publicKey" diff --git a/charts/hub/trustee/templates/kbs-route.yaml b/charts/hub/trustee/templates/kbs-route.yaml index 448f7161..7f62e7f2 100644 --- a/charts/hub/trustee/templates/kbs-route.yaml +++ b/charts/hub/trustee/templates/kbs-route.yaml @@ -13,5 +13,4 @@ spec: name: kbs-service weight: 100 tls: - termination: edge - insecureEdgeTerminationPolicy: Redirect + termination: passthrough diff --git a/charts/hub/trustee/templates/kbs.yaml b/charts/hub/trustee/templates/kbs.yaml index 661755eb..7ede92a1 100644 --- a/charts/hub/trustee/templates/kbs.yaml +++ b/charts/hub/trustee/templates/kbs.yaml @@ -9,8 +9,8 @@ spec: kbsDeploymentType: AllInOneDeployment kbsRvpsRefValuesConfigMapName: rvps-reference-values kbsSecretResources: ["kbsres1", "passphrase", "security-policy"] -# kbsHttpsKeySecretName: kbs-https-key -# kbsHttpsCertSecretName: kbs-https-certificate + kbsHttpsKeySecretName: kbs-https-key + kbsHttpsCertSecretName: kbs-https-certificate kbsResourcePolicyConfigMapName: resource-policy # TDX specific configuration (optional) diff --git a/charts/hub/trustee/templates/push-secret.yaml b/charts/hub/trustee/templates/push-secret.yaml new file mode 100644 index 00000000..c970b27e --- /dev/null +++ b/charts/hub/trustee/templates/push-secret.yaml @@ -0,0 +1,27 @@ +--- +apiVersion: external-secrets.io/v1alpha1 +kind: PushSecret +metadata: + name: push-certs + namespace: imperative +spec: + updatePolicy: Replace # Policy to overwrite existing secrets in the provider on sync + deletionPolicy: Delete # the provider' secret will be deleted if the PushSecret is deleted + refreshInterval: 10s # Refresh interval for which push secret will reconcile + secretStoreRefs: # A list of secret stores to push secrets to + - name: {{ .Values.secretStore.name }} + kind: {{ .Values.secretStore.kind }} + selector: + secret: + name: kbs-tls-self-signed # Source Kubernetes secret to be pushed + data: + - match: + secretKey: tls.key # Source Kubernetes secret key to be pushed + remoteRef: + remoteKey: "pushsecrets/kbs-tls-self-signed" # Remote reference (where the secret is going to be pushed) + property: key + - match: + secretKey: tls.crt # Source Kubernetes secret key to be pushed + remoteRef: + remoteKey: "pushsecrets/kbs-tls-self-signed" + property: certificate # Remote reference (where the secret is going to be pushed diff --git a/charts/hub/trustee/templates/tls-cert-eso.yaml b/charts/hub/trustee/templates/tls-cert-eso.yaml new file mode 100644 index 00000000..2ddb1b2e --- /dev/null +++ b/charts/hub/trustee/templates/tls-cert-eso.yaml @@ -0,0 +1,24 @@ +--- +apiVersion: "external-secrets.io/v1beta1" +kind: ExternalSecret +metadata: + annotations: + argocd.argoproj.io/sync-wave: "1" + name: tls-cert-eso + namespace: trustee-operator-system +spec: + refreshInterval: 15s + secretStoreRef: + name: {{ .Values.secretStore.name }} + kind: {{ .Values.secretStore.kind }} + target: + name: kbs-https-certificate + template: + type: Opaque + data: + - secretKey: tls.crt + remoteRef: + key: 'secret/data/pushsecrets/kbs-tls-self-signed' + property: certificate + + diff --git a/charts/hub/trustee/templates/tls-key-eso.yaml b/charts/hub/trustee/templates/tls-key-eso.yaml new file mode 100644 index 00000000..8ec36cd2 --- /dev/null +++ b/charts/hub/trustee/templates/tls-key-eso.yaml @@ -0,0 +1,24 @@ +--- +apiVersion: "external-secrets.io/v1beta1" +kind: ExternalSecret +metadata: + annotations: + argocd.argoproj.io/sync-wave: "1" + name: tls-key-eso + namespace: trustee-operator-system +spec: + refreshInterval: 15s + secretStoreRef: + name: {{ .Values.secretStore.name }} + kind: {{ .Values.secretStore.kind }} + target: + name: kbs-https-key + template: + type: Opaque + data: + - secretKey: tls.key + remoteRef: + key: 'secret/data/pushsecrets/kbs-tls-self-signed' + property: key + + diff --git a/values-simple.yaml b/values-simple.yaml index e6f87fb9..7469a8ef 100644 --- a/values-simple.yaml +++ b/values-simple.yaml @@ -35,8 +35,6 @@ clusterGroup: channel: stable installPlanApproval: Manual csv: trustee-operator.v0.4.1 - - cert-manager: name: openshift-cert-manager-operator namespace: cert-manager-operator @@ -88,6 +86,8 @@ clusterGroup: project: sandbox path: charts/coco-supported/sandbox + # Letsencrypt is not required anymore for trustee. + # It's only here if you need it for your needs. letsencrypt: name: letsencrypt namespace: letsencrypt @@ -131,6 +131,10 @@ clusterGroup: playbook: ansible/azure-nat-gateway.yaml verbosity: -vvv timeout: 3600 + - name: gen-certificate + playbook: ansible/gen-certificate.yaml + verbosity: -vvv + timeout: 3600 - name: init-data-gzipper playbook: ansible/init-data-gzipper.yaml verbosity: -vvv From f584515c11f9813f45876fcf7746cbc7c222b187 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Mon, 1 Sep 2025 17:03:03 +1000 Subject: [PATCH 2/6] chore: ansible linting Signed-off-by: Chris Butler --- requirements.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.yml b/requirements.yml index 858032f2..232e956f 100644 --- a/requirements.yml +++ b/requirements.yml @@ -2,4 +2,5 @@ collections: - azure.azcollection # Modules installed by default in the utility container, required for linting - community.general + - community.crypto - kubernetes.core From bff017ea8864e85614f1a184b209fcece605f92c Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Mon, 1 Sep 2025 17:05:43 +1000 Subject: [PATCH 3/6] chore: ansible docs Signed-off-by: Chris Butler --- ansible/gen-certificate.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/ansible/gen-certificate.yaml b/ansible/gen-certificate.yaml index 656993a4..25818d99 100644 --- a/ansible/gen-certificate.yaml +++ b/ansible/gen-certificate.yaml @@ -7,7 +7,6 @@ vars: kubeconfig: "{{ lookup('env', 'KUBECONFIG') }}" hub_domain: "{{ global.hubClusterDomain | default('none') | lower}}" - namespace: trustee-operator-system secret_name: kbs-tls-self-signed common_name: "kbs-trustee-operator-system.{{ hub_domain }}" days_valid: 365 @@ -137,4 +136,3 @@ path: "{{ tmpdir.path }}" state: absent when: need_new_cert and tmpdir is defined - From d6eb617739cf268eee59a622d3d003f348e2884c Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Mon, 1 Sep 2025 17:10:25 +1000 Subject: [PATCH 4/6] fix: linting Signed-off-by: Chris Butler --- ansible/gen-certificate.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ansible/gen-certificate.yaml b/ansible/gen-certificate.yaml index 25818d99..756bab6b 100644 --- a/ansible/gen-certificate.yaml +++ b/ansible/gen-certificate.yaml @@ -45,6 +45,7 @@ ansible.builtin.copy: content: "{{ existing_cert_data }}" dest: "{{ temp_cert_file.path }}" + mode: "0600" when: existing_secret.resources | length > 0 - name: Get certificate expiry date @@ -61,7 +62,7 @@ - name: Set fact to generate new certificate if expiring soon ansible.builtin.set_fact: need_new_cert: true - when: + when: - existing_secret.resources | length > 0 - days_until_expiry | int <= renewal_threshold_days @@ -74,7 +75,7 @@ - name: Display certificate status ansible.builtin.debug: msg: > - Certificate status: + Certificate status: {% if existing_secret.resources | length == 0 %} No existing certificate found. Will generate new certificate. {% elif need_new_cert %} From 03ebc1703d3defb850a5b74e2ae7340ea7d782d8 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Mon, 15 Sep 2025 17:57:42 +0900 Subject: [PATCH 5/6] fix: correct date time formatting Signed-off-by: Chris Butler --- ansible/gen-certificate.yaml | 2 +- ansible/init-data-gzipper.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/gen-certificate.yaml b/ansible/gen-certificate.yaml index 756bab6b..b676388c 100644 --- a/ansible/gen-certificate.yaml +++ b/ansible/gen-certificate.yaml @@ -56,7 +56,7 @@ - name: Calculate days until expiry ansible.builtin.set_fact: - days_until_expiry: "{{ ((cert_info.not_after | to_datetime('%Y%m%d%H%M%SZ')) - ansible_date_time.epoch | int | to_datetime('%s')).days }}" + days_until_expiry: "{{ ((cert_info.not_after | to_datetime('%Y%m%d%H%M%SZ')) - now()).days }}" when: existing_secret.resources | length > 0 - name: Set fact to generate new certificate if expiring soon diff --git a/ansible/init-data-gzipper.yaml b/ansible/init-data-gzipper.yaml index 714d0490..4e5b2ccd 100644 --- a/ansible/init-data-gzipper.yaml +++ b/ansible/init-data-gzipper.yaml @@ -1,4 +1,4 @@ -- name: Collect AWS facts and set secrurity group policies +- name: gzip initdata become: false connection: local hosts: localhost From a500f1ba97a7e1fdfaf5e5c0078e081e8b2983b2 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Mon, 15 Sep 2025 18:16:16 +0900 Subject: [PATCH 6/6] chore: more ansible linting Signed-off-by: Chris Butler --- ansible/init-data-gzipper.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/init-data-gzipper.yaml b/ansible/init-data-gzipper.yaml index 4e5b2ccd..20459b84 100644 --- a/ansible/init-data-gzipper.yaml +++ b/ansible/init-data-gzipper.yaml @@ -1,4 +1,4 @@ -- name: gzip initdata +- name: Gzip initdata become: false connection: local hosts: localhost