diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 092f9d66..e58a73f5 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -50,7 +50,7 @@ jobs: - name: "cargo clippy (warnings)" run: cargo clippy --all-targets -- -D warnings - name: "gofmt (check)" - run: gofmt -l . + run: if [ -n "$(gofmt -l .)" ]; then exit 1; fi - name: "go vet" run: go vet ./... - name: "Ensure Rust & Go conditions definitions use same strings" diff --git a/Cargo.lock b/Cargo.lock index 430d544c..1cc4d4f8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -415,7 +415,6 @@ name = "compute-pcrs" version = "0.1.0" dependencies = [ "anyhow", - "chrono", "clap", "compute-pcrs-lib", "k8s-openapi", @@ -2345,7 +2344,6 @@ version = "0.1.0" dependencies = [ "anyhow", "base64 0.22.1", - "chrono", "clevis-pin-trustee-lib", "compute-pcrs-lib", "env_logger", @@ -3768,7 +3766,6 @@ dependencies = [ name = "trusted-cluster-operator-lib" version = "0.1.0" dependencies = [ - "chrono", "compute-pcrs-lib", "k8s-openapi", "kube", diff --git a/Cargo.toml b/Cargo.toml index 8faec0bd..146b30ed 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,6 @@ rust-version = "1.85" [workspace.dependencies] anyhow = "1.0.100" -chrono = "0.4.42" clap = { version = "4.5.52", features = ["derive"] } clevis-pin-trustee-lib = { git = "https://github.com/latchset/clevis-pin-trustee" } compute-pcrs-lib = { git = "https://github.com/trusted-execution-clusters/compute-pcrs" } diff --git a/Makefile b/Makefile index 95fbb186..2afe9485 100644 --- a/Makefile +++ b/Makefile @@ -127,7 +127,7 @@ clean: fmt-check: cargo fmt -- --check - gofmt -l . + if [ "$$(gofmt -l .)" ]; then exit 1; fi clippy: crds-rs cargo clippy --all-targets --all-features -- -D warnings diff --git a/api/trusted-cluster-gen.go b/api/trusted-cluster-gen.go index f047e87d..c71b7fc4 100644 --- a/api/trusted-cluster-gen.go +++ b/api/trusted-cluster-gen.go @@ -138,9 +138,9 @@ func generateTrustedExecutionClusterCR(args *Args) error { Namespace: args.namespace, }, Spec: v1alpha1.TrustedExecutionClusterSpec{ - TrusteeImage: &args.trusteeImage, - PcrsComputeImage: &args.pcrsComputeImage, - RegisterServerImage: &args.registerServerImage, + TrusteeImage: args.trusteeImage, + PcrsComputeImage: args.pcrsComputeImage, + RegisterServerImage: args.registerServerImage, PublicTrusteeAddr: nil, TrusteeKbsPort: 0, RegisterServerPort: 0, @@ -176,7 +176,7 @@ func generateApprovedImageCR(args *Args) error { Namespace: args.namespace, }, Spec: v1alpha1.ApprovedImageSpec{ - Reference: &args.approvedImage, + Reference: args.approvedImage, }, } diff --git a/api/v1alpha1/conditions.go b/api/v1alpha1/conditions.go index 7107950e..a5854313 100644 --- a/api/v1alpha1/conditions.go +++ b/api/v1alpha1/conditions.go @@ -5,19 +5,19 @@ package v1alpha1 const ( - InstalledCondition string = "Installed" - InstalledReason string = "InstallationCompleted" - NotInstalledReasonNonUnique string = "NonUnique" - NotInstalledReasonInstalling string = "Installing" + InstalledCondition string = "Installed" + InstalledReason string = "InstallationCompleted" + NotInstalledReasonNonUnique string = "NonUnique" + NotInstalledReasonInstalling string = "Installing" NotInstalledReasonUninstalling string = "Uninstalling" KnownTrusteeAddressCondition string = "KnownTrusteeAddress" - KnownTrusteeAddressReason string = "AddressFound" - UnknownTrusteeAddressReason string = "NoAddressFound" + KnownTrusteeAddressReason string = "AddressFound" + UnknownTrusteeAddressReason string = "NoAddressFound" - CommittedCondition string = "Committed" - CommittedReason string = "ImageCommitted" + CommittedCondition string = "Committed" + CommittedReason string = "ImageCommitted" NotCommittedReasonComputing string = "Computing" - NotCommittedReasonNoDigest string = "NoDigestGiven" - NotCommittedReasonFailed string = "ComputationFailed" + NotCommittedReasonNoDigest string = "NoDigestGiven" + NotCommittedReasonFailed string = "ComputationFailed" ) diff --git a/api/v1alpha1/crds.go b/api/v1alpha1/crds.go index 58b9971c..8b395b5b 100644 --- a/api/v1alpha1/crds.go +++ b/api/v1alpha1/crds.go @@ -9,9 +9,9 @@ package v1alpha1 import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "sigs.k8s.io/controller-runtime/pkg/scheme" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) var ( @@ -32,7 +32,7 @@ var ( // +kubebuilder:rbac:groups=batch,resources=jobs,verbs=create;delete;list;watch // +kubebuilder:rbac:groups=trusted-execution-clusters.io,resources=trustedexecutionclusters,verbs=list;watch // +kubebuilder:rbac:groups=trusted-execution-clusters.io,resources=trustedexecutionclusters/status,verbs=patch -// +kubebuilder:rbac:groups=trusted-execution-clusters.io,resources=machines,verbs=create;list;delete;watch +// +kubebuilder:rbac:groups=trusted-execution-clusters.io,resources=machines,verbs=create;list;delete;watch;patch // +kubebuilder:rbac:groups=trusted-execution-clusters.io,resources=approvedimages,verbs=get;list;watch;patch // +kubebuilder:rbac:groups=trusted-execution-clusters.io,resources=approvedimages/status,verbs=patch @@ -40,30 +40,30 @@ var ( // +kubebuilder:validation:XValidation:rule="!has(oldSelf.publicTrusteeAddr) || has(self.publicTrusteeAddr)", message="Value is required once set" type TrustedExecutionClusterSpec struct { // Image reference to Trustee all-in-one image - // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable" - TrusteeImage *string `json:"trusteeImage"` + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable" + TrusteeImage string `json:"trusteeImage"` // Image reference to trusted-cluster-operator's compute-pcrs image - // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable" - PcrsComputeImage *string `json:"pcrsComputeImage"` + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable" + PcrsComputeImage string `json:"pcrsComputeImage"` // Image reference to trusted-cluster-operator's register-server image - // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable" - RegisterServerImage *string `json:"registerServerImage"` + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable" + RegisterServerImage string `json:"registerServerImage"` // Address where attester can connect to Trustee // +optional - // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable" + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable" PublicTrusteeAddr *string `json:"publicTrusteeAddr,omitempty"` // Port that Trustee serves on // +optional - // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable" + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable" TrusteeKbsPort int32 `json:"trusteeKbsPort,omitempty"` // Port that trusted-cluster-operator's register-server serves on // +optional - // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable" + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable" RegisterServerPort int32 `json:"registerServerPort,omitempty"` } @@ -107,9 +107,8 @@ type TrustedExecutionClusterList struct { // MachineSpec defines the desired state of Machine type MachineSpec struct { // Machine ID, typically a UUID - Id *string `json:"id"` - // Machine address - Address *string `json:"address"` + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable" + Id string `json:"id"` } // MachineStatus defines the observed state of Machine. @@ -118,6 +117,9 @@ type MachineStatus struct { // +listMapKey=type // +optional Conditions []metav1.Condition `json:"conditions,omitempty"` + // Machine address + // +optional + Address *string `json:"address,omitempty"` } // +kubebuilder:object:root=true @@ -155,7 +157,7 @@ type ApprovedImageSpec struct { // +required // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable" // +kubebuilder:validation:XValidation:rule="self.matches(r'.*@sha256:.*')",message="Image must be provided with a digest" - Reference *string `json:"image"` + Reference string `json:"image"` } // ApprovedImageStatus defines the observed state of ApprovedImage. diff --git a/compute-pcrs/Cargo.toml b/compute-pcrs/Cargo.toml index ceab8c8c..b983a3e2 100644 --- a/compute-pcrs/Cargo.toml +++ b/compute-pcrs/Cargo.toml @@ -12,7 +12,6 @@ description = "A trusted-cluster-operator optimized compute-pcrs interface" [dependencies] anyhow.workspace = true -chrono.workspace = true clap.workspace = true trusted-cluster-operator-lib = { path = "../lib" } compute-pcrs-lib.workspace = true diff --git a/compute-pcrs/src/main.rs b/compute-pcrs/src/main.rs index 4a7e11ef..2509e2c0 100644 --- a/compute-pcrs/src/main.rs +++ b/compute-pcrs/src/main.rs @@ -4,10 +4,9 @@ // SPDX-License-Identifier: MIT use anyhow::{Context, Result}; -use chrono::Utc; use clap::Parser; use compute_pcrs_lib::*; -use k8s_openapi::api::core::v1::ConfigMap; +use k8s_openapi::{api::core::v1::ConfigMap, chrono::Utc}; use kube::{Api, Client}; use trusted_cluster_operator_lib::{conditions::INSTALLED_REASON, reference_values::*, *}; diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index 7e579ae4..caf9d9a8 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -31,7 +31,6 @@ resources: - trustedexecutioncluster_editor_role.yaml - trustedexecutioncluster_viewer_role.yaml - machine_admin_role.yaml - - machine_editor_role.yaml - machine_viewer_role.yaml - approvedimage_admin_role.yaml - approvedimage_viewer_role.yaml diff --git a/config/rbac/machine_editor_role.yaml b/config/rbac/machine_editor_role.yaml deleted file mode 100644 index cefd404a..00000000 --- a/config/rbac/machine_editor_role.yaml +++ /dev/null @@ -1,37 +0,0 @@ -# SPDX-FileCopyrightText: Generated by kubebuilder -# -# SPDX-License-Identifier: CC0-1.0 - -# This rule is not used by the project trusted-cluster-operator itself. -# It is provided to allow the cluster admin to help manage permissions for users. -# -# Grants permissions to create, update, and delete resources within the trusted-execution-clusters.io. -# This role is intended for users who need to manage these resources -# but should not control RBAC or manage permissions for others. - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - app.kubernetes.io/name: trusted-cluster-operator - app.kubernetes.io/managed-by: kustomize - name: machine-editor-role -rules: -- apiGroups: - - trusted-execution-clusters.io - resources: - - machines - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - trusted-execution-clusters.io - resources: - - machines/status - verbs: - - get diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 88a7a257..8419e54e 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -11,7 +11,6 @@ edition.workspace = true rust-version.workspace = true [dependencies] -chrono.workspace = true compute-pcrs-lib.workspace = true k8s-openapi.workspace = true kube.workspace = true diff --git a/lib/src/lib.rs b/lib/src/lib.rs index f9f94434..65966e3a 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -10,9 +10,9 @@ pub use kopium::approvedimages::*; pub use kopium::machines::*; pub use kopium::trustedexecutionclusters::*; -use chrono::Utc; use conditions::*; use k8s_openapi::apimachinery::pkg::apis::meta::v1::{Condition, Time}; +use k8s_openapi::chrono::Utc; #[macro_export] macro_rules! update_status { diff --git a/lib/src/reference_values.rs b/lib/src/reference_values.rs index a9e7be47..0e6361f4 100644 --- a/lib/src/reference_values.rs +++ b/lib/src/reference_values.rs @@ -3,8 +3,8 @@ // // SPDX-License-Identifier: MIT -use chrono::{DateTime, Utc}; use compute_pcrs_lib::Pcr; +use k8s_openapi::chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; diff --git a/operator/Cargo.toml b/operator/Cargo.toml index 0d7c7bbc..ba6cf1f7 100644 --- a/operator/Cargo.toml +++ b/operator/Cargo.toml @@ -11,7 +11,6 @@ rust-version.workspace = true [dependencies] anyhow.workspace = true base64 = "0.22.1" -chrono.workspace = true clevis-pin-trustee-lib.workspace = true trusted-cluster-operator-lib = { path = "../lib" } compute-pcrs-lib.workspace = true diff --git a/operator/src/conditions.rs b/operator/src/conditions.rs index 43e31c60..ba397e15 100644 --- a/operator/src/conditions.rs +++ b/operator/src/conditions.rs @@ -2,8 +2,8 @@ // // SPDX-License-Identifier: MIT -use chrono::Utc; use k8s_openapi::apimachinery::pkg::apis::meta::v1::{Condition, Time}; +use k8s_openapi::chrono::Utc; use trusted_cluster_operator_lib::{condition_status, conditions::*}; pub fn known_trustee_address_condition(known: bool, generation: Option) -> Condition { diff --git a/operator/src/lib.rs b/operator/src/lib.rs index f0994bc0..af0effd9 100644 --- a/operator/src/lib.rs +++ b/operator/src/lib.rs @@ -8,8 +8,9 @@ // // Use in other crates is not an intended purpose. +use anyhow::Context; use k8s_openapi::apimachinery::pkg::apis::meta::v1::OwnerReference; -use kube::{Client, runtime::controller::Action}; +use kube::{Client, Resource, runtime::controller::Action}; use log::info; use std::fmt::{Debug, Display}; use std::{sync::Arc, time::Duration}; @@ -53,3 +54,19 @@ macro_rules! create_or_info_if_exists { } }; } + +pub fn generate_owner_reference>( + object: &T, +) -> anyhow::Result { + let name = object.meta().name.clone(); + let uid = object.meta().uid.clone(); + let kind = T::kind(&()).to_string(); + Ok(OwnerReference { + api_version: T::api_version(&()).to_string(), + block_owner_deletion: Some(true), + controller: Some(true), + name: name.context(format!("{} had no name", kind.clone()))?, + uid: uid.context(format!("{} had no UID", kind.clone()))?, + kind, + }) +} diff --git a/operator/src/main.rs b/operator/src/main.rs index 91f35cd5..27f4c6f1 100644 --- a/operator/src/main.rs +++ b/operator/src/main.rs @@ -6,19 +6,16 @@ use std::sync::Arc; use std::time::Duration; -use anyhow::{Context, Result}; +use anyhow::Result; use env_logger::Env; use futures_util::StreamExt; -use k8s_openapi::apimachinery::pkg::apis::meta::v1::{Condition, OwnerReference}; -use kube::{Api, Client, Resource}; -use kube::{ - api::ObjectMeta, - runtime::{ - controller::{Action, Controller}, - watcher, - }, -}; +use k8s_openapi::apimachinery::pkg::apis::meta::v1::Condition; +use kube::runtime::controller::{Action, Controller}; +use kube::runtime::watcher; +use kube::{Api, Client}; use log::{error, info, warn}; + +use operator::generate_owner_reference; use trusted_cluster_operator_lib::{TrustedExecutionCluster, TrustedExecutionClusterStatus}; use trusted_cluster_operator_lib::{conditions::*, update_status}; @@ -97,24 +94,11 @@ async fn reconcile( Ok(Action::await_change()) } -fn generate_owner_reference(metadata: &ObjectMeta) -> Result { - let name = metadata.name.clone(); - let uid = metadata.uid.clone(); - Ok(OwnerReference { - api_version: TrustedExecutionCluster::api_version(&()).to_string(), - block_owner_deletion: Some(true), - controller: Some(true), - kind: TrustedExecutionCluster::kind(&()).to_string(), - name: name.context("TrustedExecutionCluster had no name")?, - uid: uid.context("TrustedExecutionCluster had no UID")?, - }) -} - async fn install_trustee_configuration( client: Client, cluster: &TrustedExecutionCluster, ) -> Result<()> { - let owner_reference = generate_owner_reference(&cluster.metadata)?; + let owner_reference = generate_owner_reference(cluster)?; match trustee::generate_trustee_data(client.clone(), owner_reference.clone()).await { Ok(_) => info!("Generate configmap for the KBS configuration",), @@ -154,7 +138,7 @@ async fn install_trustee_configuration( } async fn install_register_server(client: Client, cluster: &TrustedExecutionCluster) -> Result<()> { - let owner_reference = generate_owner_reference(&cluster.metadata)?; + let owner_reference = generate_owner_reference(cluster)?; match register_server::create_register_server_deployment( client.clone(), @@ -199,10 +183,10 @@ async fn main() -> Result<()> { #[cfg(test)] mod tests { - use chrono::Utc; use http::{Method, Request, StatusCode}; - use k8s_openapi::apimachinery::pkg::apis::meta::v1::Time; - use kube::{api::ObjectList, client::Body}; + use k8s_openapi::{apimachinery::pkg::apis::meta::v1::Time, chrono::Utc}; + use kube::api::{ObjectList, ObjectMeta}; + use kube::client::Body; use trusted_cluster_operator_lib::TrustedExecutionClusterSpec; use super::*; diff --git a/operator/src/mock_client.rs b/operator/src/mock_client.rs index 24c59fb8..98d30963 100644 --- a/operator/src/mock_client.rs +++ b/operator/src/mock_client.rs @@ -3,10 +3,9 @@ // // SPDX-License-Identifier: MIT -use chrono::Utc; use compute_pcrs_lib::Pcr; use http::{Method, Request, Response, StatusCode}; -use k8s_openapi::api::core::v1::ConfigMap; +use k8s_openapi::{api::core::v1::ConfigMap, chrono::Utc}; use kube::{Client, client::Body, error::ErrorResponse}; use operator::RvContextData; use serde::Serialize; diff --git a/operator/src/reference_values.rs b/operator/src/reference_values.rs index 12afb301..315e51d1 100644 --- a/operator/src/reference_values.rs +++ b/operator/src/reference_values.rs @@ -4,7 +4,6 @@ // SPDX-License-Identifier: MIT use anyhow::{Context, Result, anyhow}; -use chrono::Utc; use compute_pcrs_lib::Pcr; use futures_util::StreamExt; use k8s_openapi::{ @@ -16,6 +15,7 @@ use k8s_openapi::{ }, }, apimachinery::pkg::apis::meta::v1::OwnerReference, + chrono::Utc, }; use kube::api::{DeleteParams, ObjectMeta}; use kube::runtime::{ diff --git a/operator/src/register_server.rs b/operator/src/register_server.rs index 6fd028d1..7503fe9d 100644 --- a/operator/src/register_server.rs +++ b/operator/src/register_server.rs @@ -3,7 +3,7 @@ // // SPDX-License-Identifier: MIT -use anyhow::Result; +use anyhow::{Result, anyhow}; use futures_util::StreamExt; use k8s_openapi::{ api::{ @@ -19,17 +19,20 @@ use k8s_openapi::{ }; use kube::runtime::{ controller::{Action, Controller}, - watcher, + finalizer, + finalizer::Event, }; use kube::{Api, Client, Resource}; use log::info; use std::{collections::BTreeMap, sync::Arc}; use crate::trustee; -use operator::{ControllerError, controller_error_policy, create_or_info_if_exists}; +use operator::*; use trusted_cluster_operator_lib::Machine; const INTERNAL_REGISTER_SERVER_PORT: i32 = 8000; +/// Finalizer name to discard decryption keys when a machine is deleted +const MACHINE_FINALIZER: &str = "finalizer.machine.trusted-execution-clusters.io"; pub async fn create_register_server_deployment( client: Client, @@ -125,24 +128,41 @@ async fn keygen_reconcile( machine: Arc, client: Arc, ) -> Result { - let client = Arc::unwrap_or_clone(client); - let id = &machine.spec.id; - trustee::generate_secret(client.clone(), id).await?; - trustee::mount_secret(client.clone(), id).await?; - Ok(Action::await_change()) + let machines: Api = Api::default_namespaced(Arc::unwrap_or_clone(client.clone())); + finalizer(&machines, MACHINE_FINALIZER, machine, |ev| async move { + match ev { + Event::Apply(machine) => { + let kube_client = Arc::unwrap_or_clone(client); + let id = &machine.spec.id.clone(); + async { + let owner_reference = generate_owner_reference(&Arc::unwrap_or_clone(machine))?; + trustee::generate_secret(kube_client.clone(), id, owner_reference).await?; + trustee::mount_secret(kube_client, id).await + } + .await + .map(|_| Action::await_change()) + .map_err(|e| finalizer::Error::::ApplyFailed(e.into())) + } + Event::Cleanup(machine) => { + let kube_client = Arc::unwrap_or_clone(client); + let id = &machine.spec.id; + trustee::unmount_secret(kube_client, id) + .await + .map(|_| Action::await_change()) + .map_err(|e| finalizer::Error::::CleanupFailed(e.into())) + } + } + }) + .await + .map_err(|e| anyhow!("failed to reconcile on machine: {e}").into()) } pub async fn launch_keygen_controller(client: Client) { let machines: Api = Api::default_namespaced(client.clone()); tokio::spawn( - Controller::new(machines, watcher::Config::default()) + Controller::new(machines, Default::default()) .run(keygen_reconcile, controller_error_policy, Arc::new(client)) - .for_each(|res| async move { - match res { - Ok(o) => info!("reconciled {o:?}"), - Err(e) => info!("reconcile failed: {e:?}"), - } - }), + .for_each(controller_info), ); } diff --git a/operator/src/trustee.rs b/operator/src/trustee.rs index ba384ab7..624ac34d 100644 --- a/operator/src/trustee.rs +++ b/operator/src/trustee.rs @@ -6,7 +6,6 @@ use anyhow::{Context, Result}; use base64::{Engine as _, engine::general_purpose}; -use chrono::{DateTime, TimeDelta, Utc}; use clevis_pin_trustee_lib::Key as ClevisKey; use k8s_openapi::api::apps::v1::{Deployment, DeploymentSpec}; use k8s_openapi::api::core::v1::{ @@ -18,6 +17,7 @@ use k8s_openapi::apimachinery::pkg::{ apis::meta::v1::{LabelSelector, OwnerReference}, util::intstr::IntOrString, }; +use k8s_openapi::chrono::{DateTime, TimeDelta, Utc}; use kube::{Api, Client, Resource, api::ObjectMeta}; use log::info; use operator::{RvContextData, create_or_info_if_exists}; @@ -134,34 +134,67 @@ fn generate_secret_volume(id: &str) -> (Volume, VolumeMount) { } pub async fn mount_secret(client: Client, id: &str) -> Result<()> { + let result = do_mount_secret(client, id, true).await; + info!("Mounted secret {id} to {DEPLOYMENT_NAME}"); + result +} + +pub async fn unmount_secret(client: Client, id: &str) -> Result<()> { + let result = do_mount_secret(client, id, false).await; + info!("Unmounted secret {id} from {DEPLOYMENT_NAME}"); + result +} + +pub async fn do_mount_secret(client: Client, id: &str, add: bool) -> Result<()> { let deployments: Api = Api::default_namespaced(client); let mut deployment = deployments.get(DEPLOYMENT_NAME).await?; let err = format!("Deployment {DEPLOYMENT_NAME} existed, but had no spec"); let depl_spec = deployment.spec.as_mut().context(err)?; let err = format!("Deployment {DEPLOYMENT_NAME} existed, but had no pod spec"); let pod_spec = depl_spec.template.spec.as_mut().context(err)?; - - let (volume, volume_mount) = generate_secret_volume(id); - pod_spec.volumes.get_or_insert_default().push(volume); let err = format!("Deployment {DEPLOYMENT_NAME} existed, but had no containers"); let container = pod_spec.containers.get_mut(0).context(err)?; let vol_mounts = container.volume_mounts.get_or_insert_default(); - vol_mounts.push(volume_mount); + + if add { + let (volume, volume_mount) = generate_secret_volume(id); + pod_spec.volumes.get_or_insert_default().push(volume); + vol_mounts.push(volume_mount); + } else { + let vol_result = pod_spec.volumes.as_mut().and_then(|vs| { + let pos = vs.iter().position(|v| v.name == id); + pos.map(|p| vs.swap_remove(p)) + }); + if vol_result.is_none() { + info!("Secret {id} was to be dropped, but volume had already been removed"); + } + let vol_mount_result = container.volume_mounts.as_mut().and_then(|vms| { + let pos = vms.iter().position(|v| v.name == id); + pos.map(|p| vms.swap_remove(p)) + }); + if vol_mount_result.is_none() { + info!("Secret {id} was to be dropped, but volume mount had already been removed"); + } + } deployments .replace(DEPLOYMENT_NAME, &Default::default(), &deployment) .await?; - info!("Mounted secret {id} to {DEPLOYMENT_NAME}"); Ok(()) } -pub async fn generate_secret(client: Client, id: &str) -> Result<()> { +pub async fn generate_secret( + client: Client, + id: &str, + owner_reference: OwnerReference, +) -> Result<()> { let secret_data = k8s_openapi::ByteString(generate_luks_key()?); let data = BTreeMap::from([("root".to_string(), secret_data)]); let secret = Secret { metadata: ObjectMeta { name: Some(id.to_string()), + owner_references: Some(vec![owner_reference]), ..Default::default() }, data: Some(data), @@ -369,6 +402,7 @@ mod tests { use crate::mock_client::*; use compute_pcrs_lib::Pcr; use http::{Method, Request, StatusCode}; + use kube::client::Body; fn dummy_pcrs() -> ImagePcrs { ImagePcrs(BTreeMap::from([( @@ -598,6 +632,37 @@ mod tests { }); } + #[tokio::test] + async fn test_unmount_secret() { + let clos = async |req: Request, ctr| match (ctr, req.method()) { + (0, &Method::GET) => { + let mut depl = dummy_deployment(); + let spec = depl.spec.as_mut().unwrap(); + let pod_spec = spec.template.spec.as_mut().unwrap(); + pod_spec.volumes = Some(vec![Volume { + name: "id".to_string(), + ..Default::default() + }]); + let container = pod_spec.containers.get_mut(0).unwrap(); + container.volume_mounts = Some(vec![VolumeMount { + name: "id".to_string(), + ..Default::default() + }]); + Ok(serde_json::to_string(&depl).unwrap()) + } + (1, &Method::PUT) => { + let bytes = req.into_body().collect_bytes().await.unwrap().to_vec(); + let body = String::from_utf8_lossy(&bytes); + assert!(!body.contains("id")); + Ok(serde_json::to_string(&dummy_deployment()).unwrap()) + } + _ => panic!("unexpected API interaction: {req:?}, counter {ctr}"), + }; + count_check!(2, clos, |client| { + assert!(unmount_secret(client, "id").await.is_ok()); + }); + } + #[tokio::test] async fn test_generate_att_policy_success() { let clos = |client| generate_attestation_policy(client, Default::default()); @@ -618,19 +683,19 @@ mod tests { #[tokio::test] async fn test_generate_secret_success() { - let clos = |client| generate_secret(client, "id"); + let clos = |client| generate_secret(client, "id", Default::default()); test_create_success::<_, _, Secret>(clos).await; } #[tokio::test] async fn test_generate_secret_already_exists() { - let clos = |client| generate_secret(client, "id"); + let clos = |client| generate_secret(client, "id", Default::default()); test_create_already_exists(clos).await; } #[tokio::test] async fn test_generate_secret_error() { - let clos = |client| generate_secret(client, "id"); + let clos = |client| generate_secret(client, "id", Default::default()); test_create_error(clos).await; } diff --git a/register-server/src/main.rs b/register-server/src/main.rs index 7a9426ee..3660abc4 100644 --- a/register-server/src/main.rs +++ b/register-server/src/main.rs @@ -18,7 +18,7 @@ use std::net::SocketAddr; use uuid::Uuid; use warp::{http::StatusCode, reply, Filter}; -use trusted_cluster_operator_lib::{Machine, MachineSpec, TrustedExecutionCluster}; +use trusted_cluster_operator_lib::*; #[derive(Parser)] #[command(name = "register-server")] @@ -142,7 +142,8 @@ async fn create_machine(client: Client, uuid: &str, client_ip: &str) -> anyhow:: let machine_list = machines.list(&Default::default()).await?; for existing_machine in machine_list.items { - if existing_machine.spec.address == client_ip { + let existing_address = existing_machine.status.and_then(|s| s.address); + if existing_address.map(|a| a == client_ip).unwrap_or(false) { if let Some(name) = &existing_machine.metadata.name { info!("Found existing machine {name} with IP {client_ip}, deleting..."); machines.delete(name, &Default::default()).await?; @@ -159,9 +160,11 @@ async fn create_machine(client: Client, uuid: &str, client_ip: &str) -> anyhow:: }, spec: MachineSpec { id: uuid.to_string(), - address: client_ip.to_string(), }, - status: None, + status: Some(MachineStatus { + address: Some(client_ip.to_string()), + conditions: None, + }), }; machines.create(&Default::default(), &machine).await?; diff --git a/tests/attestation.rs b/tests/attestation.rs index b582456b..64eb4005 100644 --- a/tests/attestation.rs +++ b/tests/attestation.rs @@ -7,45 +7,73 @@ use trusted_cluster_operator_test_utils::*; #[cfg(feature = "virtualization")] use trusted_cluster_operator_test_utils::virt; -virt_test! { -async fn test_attestation() -> anyhow::Result<()> { - let test_ctx = setup!().await?; - let client = test_ctx.client(); - let namespace = test_ctx.namespace(); +#[cfg(feature = "virtualization")] +struct SingleAttestationContext { + key_path: std::path::PathBuf, + vm_name: String, +} - let (_private_key, public_key, key_path) = virt::generate_ssh_key_pair()?; - test_ctx.info(format!("Generated SSH key pair and added to ssh-agent: {:?}", key_path)); +#[cfg(feature = "virtualization")] +impl SingleAttestationContext { + async fn new(test_ctx: &TestContext) -> anyhow::Result { + let client = test_ctx.client(); + let namespace = test_ctx.namespace(); + + let (_private_key, public_key, key_path) = virt::generate_ssh_key_pair()?; + test_ctx.info(format!( + "Generated SSH key pair and added to ssh-agent: {:?}", + key_path + )); + + let vm_name = "test-coreos-vm"; + let register_server_url = format!( + "http://register-server.{}.svc.cluster.local:8000/ignition-clevis-pin-trustee", + namespace + ); + let image = "quay.io/trusted-execution-clusters/fedora-coreos-kubevirt:latest"; - let vm_name = "test-coreos-vm"; - let register_server_url = format!( - "http://register-server.{}.svc.cluster.local:8000/ignition-clevis-pin-trustee", - namespace - ); - let image = "quay.io/trusted-execution-clusters/fedora-coreos-kubevirt:latest"; + test_ctx.info(format!("Creating VM: {}", vm_name)); + virt::create_kubevirt_vm( + client, + namespace, + vm_name, + &public_key, + ®ister_server_url, + image, + ) + .await?; - test_ctx.info(format!("Creating VM: {}", vm_name)); - virt::create_kubevirt_vm( - client, - namespace, - vm_name, - &public_key, - ®ister_server_url, - image, - ) - .await?; + test_ctx.info(format!("Waiting for VM {} to reach Running state", vm_name)); + virt::wait_for_vm_running(client, namespace, vm_name, 300).await?; + test_ctx.info(format!("VM {} is Running", vm_name)); - test_ctx.info(format!("Waiting for VM {} to reach Running state", vm_name)); - virt::wait_for_vm_running(client, namespace, vm_name, 300).await?; - test_ctx.info(format!("VM {} is Running", vm_name)); + test_ctx.info(format!("Waiting for SSH access to VM {}", vm_name)); + virt::wait_for_vm_ssh_ready(namespace, vm_name, &key_path, 300).await?; + test_ctx.info("SSH access is ready"); - test_ctx.info(format!("Waiting for SSH access to VM {}", vm_name)); - virt::wait_for_vm_ssh_ready(namespace, vm_name, &key_path, 300).await?; - test_ctx.info("SSH access is ready"); + Ok(Self { + key_path, + vm_name: vm_name.to_string(), + }) + } +} - test_ctx.info("Verifying encrypted root device"); - let has_encrypted_root = virt::verify_encrypted_root(namespace, vm_name, &key_path).await?; +#[cfg(feature = "virtualization")] +impl Drop for SingleAttestationContext { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.key_path); + } +} - let _ = std::fs::remove_file(&key_path); +virt_test! { +async fn test_attestation() -> anyhow::Result<()> { + let test_ctx = setup!().await?; + let att_ctx = SingleAttestationContext::new(&test_ctx).await?; + + test_ctx.info("Verifying encrypted root device"); + let namespace = test_ctx.namespace(); + let has_encrypted_root = + virt::verify_encrypted_root(namespace, &att_ctx.vm_name, &att_ctx.key_path).await?; assert!( has_encrypted_root, @@ -162,40 +190,13 @@ async fn test_parallel_vm_attestation() -> anyhow::Result<()> { virt_test! { async fn test_vm_reboot_attestation() -> anyhow::Result<()> { let test_ctx = setup!().await?; - let client = test_ctx.client(); - let namespace = test_ctx.namespace(); - test_ctx.info("Testing VM reboot - VM should successfully boot after multiple reboots"); - - let (_private_key, public_key, key_path) = virt::generate_ssh_key_pair()?; - test_ctx.info(format!("Generated SSH key pair: {:?}", key_path)); - - let vm_name = "test-coreos-reboot"; - let register_server_url = format!( - "http://register-server.{}.svc.cluster.local:8000/ignition-clevis-pin-trustee", - namespace - ); - let image = "quay.io/trusted-execution-clusters/fedora-coreos-kubevirt:latest"; - - test_ctx.info(format!("Creating VM: {}", vm_name)); - virt::create_kubevirt_vm( - client, - namespace, - vm_name, - &public_key, - ®ister_server_url, - image, - ) - .await?; - - test_ctx.info("Waiting for VM to reach Running state"); - virt::wait_for_vm_running(client, namespace, vm_name, 300).await?; - - test_ctx.info("Waiting for SSH access"); - virt::wait_for_vm_ssh_ready(namespace, vm_name, &key_path, 300).await?; + let att_ctx = SingleAttestationContext::new(&test_ctx).await?; + let namespace = test_ctx.namespace(); test_ctx.info("Verifying initial encrypted root device"); - let has_encrypted_root = virt::verify_encrypted_root(namespace, vm_name, &key_path).await?; + let has_encrypted_root = + virt::verify_encrypted_root(namespace, &att_ctx.vm_name, &att_ctx.key_path).await?; assert!( has_encrypted_root, "VM should have encrypted root device on initial boot" @@ -210,29 +211,29 @@ async fn test_vm_reboot_attestation() -> anyhow::Result<()> { // Reboot the VM via SSH let _reboot_result = virt::virtctl_ssh_exec( namespace, - vm_name, - &key_path, - "sudo systemctl reboot" - ).await; + &att_ctx.vm_name, + &att_ctx.key_path, + "sudo systemctl reboot", + ) + .await; tokio::time::sleep(std::time::Duration::from_secs(10)).await; test_ctx.info(format!("Waiting for SSH access after reboot {}", i)); - virt::wait_for_vm_ssh_ready(namespace, vm_name, &key_path, 300).await?; + virt::wait_for_vm_ssh_ready(namespace, &att_ctx.vm_name, &att_ctx.key_path, 300).await?; // Verify encrypted root is still present after reboot test_ctx.info(format!("Verifying encrypted root after reboot {}", i)); - let has_encrypted_root = virt::verify_encrypted_root(namespace, vm_name, &key_path).await?; + let has_encrypted_root = + virt::verify_encrypted_root(namespace, &att_ctx.vm_name, &att_ctx.key_path).await?; assert!( has_encrypted_root, - "VM should have encrypted root device after reboot {}", i + "VM should have encrypted root device after reboot {}", + i ); test_ctx.info(format!("Reboot {}: attestation successful", i)); } - // Clean up SSH key - let _ = std::fs::remove_file(&key_path); - test_ctx.info(format!( "VM successfully rebooted {} times with encrypted root device maintained", num_reboots @@ -243,3 +244,43 @@ async fn test_vm_reboot_attestation() -> anyhow::Result<()> { Ok(()) } } + +virt_test! { +async fn test_vm_reboot_delete_machine() -> anyhow::Result<()> { + use kube::Api; + use trusted_cluster_operator_lib::Machine; + + let test_ctx = setup!().await?; + test_ctx.info("Testing Machine deletion - VM should no longer boot successfully when its Machine CRD was removed"); + let att_ctx = SingleAttestationContext::new(&test_ctx).await?; + + let machines: Api = Api::namespaced(test_ctx.client().clone(), test_ctx.namespace()); + let list = machines.list(&Default::default()).await?; + let name = list.items[0].metadata.name.as_ref().unwrap(); + machines.delete(name, &Default::default()).await?; + + test_ctx.info("Performing reboot, expecting missing resource"); + let _reboot_result = virt::virtctl_ssh_exec( + test_ctx.namespace(), + &att_ctx.vm_name, + &att_ctx.key_path, + "sudo systemctl reboot", + ) + .await; + + tokio::time::sleep(std::time::Duration::from_secs(10)).await; + + test_ctx.info("Waiting for SSH access after machine removal"); + let wait = virt::wait_for_vm_ssh_ready( + test_ctx.namespace(), + &att_ctx.vm_name, + &att_ctx.key_path, + 300, + ) + .await; + assert!(wait.is_err()); + + test_ctx.cleanup().await?; + Ok(()) +} +}