From 8dd12faeb2d1a7d8648c34dfbd7f4f57a1eeb4ac Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Fri, 21 Nov 2025 15:41:58 +0000 Subject: [PATCH 01/52] [accless] E: Unify Headers Under Shared Dir --- accless/CMakeLists.txt | 32 ++++++++++++++++------- accless/src/accless.cpp | 14 +++++----- accless/src/dag.cpp | 2 +- accless/src/utils.cpp | 2 +- applications/CMakeLists.txt | 13 +-------- applications/test/CMakeLists.txt | 2 +- applications/test/att-client-sgx/main.cpp | 6 ++--- applications/test/att-client-snp/main.cpp | 6 ++--- 8 files changed, 40 insertions(+), 37 deletions(-) diff --git a/accless/CMakeLists.txt b/accless/CMakeLists.txt index db8914c..cc99387 100644 --- a/accless/CMakeLists.txt +++ b/accless/CMakeLists.txt @@ -65,12 +65,24 @@ add_subdirectory(./libs/jwt/cpp-bindings) add_subdirectory(./libs/abe4/cpp-bindings) add_subdirectory(./libs/base64) -set(ACCLESS_COMMON_HEADERS - ${ACCLESS_ROOT}/include - ${ACCLESS_ROOT}/libs/jwt/cpp-bindings - ${ACCLESS_ROOT}/libs/abe4/cpp-bindings - ${ACCLESS_ROOT}/libs/base64 -) +# Create a directory for namespaced headers. This allows including accless +# headers with a namespace prefix, e.g. #include "accless/attestation/attestation.h" +set(ACCLESS_INCLUDE_PREFIX_DIR ${CMAKE_CURRENT_BINARY_DIR}/include_prefix) +set(ACCLESS_INCLUDE_PREFIX_DIR ${ACCLESS_INCLUDE_PREFIX_DIR} PARENT_SCOPE) +if (EXISTS ${ACCLESS_INCLUDE_PREFIX_DIR}) + file(REMOVE_RECURSE ${ACCLESS_INCLUDE_PREFIX_DIR}) +endif() +file(MAKE_DIRECTORY ${ACCLESS_INCLUDE_PREFIX_DIR}/accless) +# Create symlinks for common libraries +set(ACCLESS_LIBS_DIR ${CMAKE_CURRENT_SOURCE_DIR}/libs) +file(CREATE_LINK ${ACCLESS_LIBS_DIR}/jwt/cpp-bindings ${ACCLESS_INCLUDE_PREFIX_DIR}/accless/jwt SYMBOLIC) +file(CREATE_LINK ${ACCLESS_LIBS_DIR}/abe4/cpp-bindings ${ACCLESS_INCLUDE_PREFIX_DIR}/accless/abe4 SYMBOLIC) +file(CREATE_LINK ${ACCLESS_LIBS_DIR}/base64 ${ACCLESS_INCLUDE_PREFIX_DIR}/accless/base64 SYMBOLIC) +# Symlink individual headers from accless/include +file(CREATE_LINK ${CMAKE_CURRENT_SOURCE_DIR}/include/accless.h ${ACCLESS_INCLUDE_PREFIX_DIR}/accless/accless.h SYMBOLIC) +file(CREATE_LINK ${CMAKE_CURRENT_SOURCE_DIR}/include/dag.h ${ACCLESS_INCLUDE_PREFIX_DIR}/accless/dag.h SYMBOLIC) +file(CREATE_LINK ${CMAKE_CURRENT_SOURCE_DIR}/include/utils.h ${ACCLESS_INCLUDE_PREFIX_DIR}/accless/utils.h SYMBOLIC) + set(ACCLESS_COMMON_LIBRARIES accless::jwt accless::abe4 accless::base64) if (CMAKE_SYSTEM_NAME STREQUAL "WASI") @@ -96,18 +108,20 @@ else () add_subdirectory(./libs/attestation) add_subdirectory(./libs/s3) + # Create symlinks for native-only libraries + file(CREATE_LINK ${ACCLESS_LIBS_DIR}/attestation ${ACCLESS_INCLUDE_PREFIX_DIR}/accless/attestation SYMBOLIC) + file(CREATE_LINK ${ACCLESS_LIBS_DIR}/s3 ${ACCLESS_INCLUDE_PREFIX_DIR}/accless/s3 SYMBOLIC) + set(ACCLESS_LIBRARIES accless::s3 accless::attestation # We use the installed curl as part of azure-cvm-attestation /usr/local/attestationcurl/lib/libcurl.a ) - set(ACCLESS_HEADERS ${ACCLESS_ROOT}/libs) endif() target_include_directories(${CMAKE_PROJECT_TARGET} PUBLIC - ${ACCLESS_COMMON_HEADERS} - ${ACCLESS_HEADERS} + ${ACCLESS_INCLUDE_PREFIX_DIR} ) target_link_libraries(${CMAKE_PROJECT_TARGET} PUBLIC ${ACCLESS_COMMON_LIBRARIES} diff --git a/accless/src/accless.cpp b/accless/src/accless.cpp index a064cd9..a23eefa 100644 --- a/accless/src/accless.cpp +++ b/accless/src/accless.cpp @@ -1,15 +1,15 @@ -#include "accless.h" -#include "base64.h" -#include "dag.h" -#include "jwt.h" -#include "utils.h" +#include "accless/accless.h" +#include "accless/base64/base64.h" +#include "accless/dag.h" +#include "accless/jwt/jwt.h" +#include "accless/utils.h" #ifdef __faasm // Faasm includes #include #else -#include "attestation/attestation.h" -#include "s3/S3Wrapper.hpp" +#include "accless/attestation/attestation.h" +#include "accless/s3/S3Wrapper.hpp" #endif #include diff --git a/accless/src/dag.cpp b/accless/src/dag.cpp index 980abb6..773850d 100644 --- a/accless/src/dag.cpp +++ b/accless/src/dag.cpp @@ -1,4 +1,4 @@ -#include "dag.h" +#include "accless/dag.h" #include #include diff --git a/accless/src/utils.cpp b/accless/src/utils.cpp index 169612d..1f008e8 100644 --- a/accless/src/utils.cpp +++ b/accless/src/utils.cpp @@ -1,4 +1,4 @@ -#include "utils.h" +#include "accless/utils.h" #ifdef __faasm extern "C" { diff --git a/applications/CMakeLists.txt b/applications/CMakeLists.txt index 8cb35a8..18c146b 100644 --- a/applications/CMakeLists.txt +++ b/applications/CMakeLists.txt @@ -22,21 +22,10 @@ add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/../accless ${ACCLESS_BUILD_DIRECTORY} # Prepare variables for workflow compilation if (CMAKE_SYSTEM_NAME STREQUAL "WASI") set(ACCLESS_LIBRARIES faasm accless::accless) - set(ACCLESS_HEADERS - ${CMAKE_CURRENT_LIST_DIR}/../accless/include - ${CMAKE_CURRENT_LIST_DIR}/../accless/libs/jwt/cpp-bindings - ${CMAKE_CURRENT_LIST_DIR}/../accless/libs/abe4/cpp-bindings - ${CMAKE_CURRENT_LIST_DIR}/../accless/libs/base64 - ) else () set(ACCLESS_LIBRARIES accless::accless) - set(ACCLESS_HEADERS - ${CMAKE_CURRENT_LIST_DIR}/../accless/include - ${CMAKE_CURRENT_LIST_DIR}/../accless/libs/jwt/cpp-bindings - ${CMAKE_CURRENT_LIST_DIR}/../accless/libs/abe4/cpp-bindings - ${CMAKE_CURRENT_LIST_DIR}/../accless/libs/base64 - ) endif() +set(ACCLESS_HEADERS ${ACCLESS_INCLUDE_PREFIX_DIR}) # ============================================================================= # Application subdirectories diff --git a/applications/test/CMakeLists.txt b/applications/test/CMakeLists.txt index f5c132d..c372446 100644 --- a/applications/test/CMakeLists.txt +++ b/applications/test/CMakeLists.txt @@ -1,3 +1,3 @@ # Mocked client replicating the behaviour of SGX-Faasm during remote attestation. add_subdirectory(./att-client-sgx) -add_subdirectory(att-client-snp) +add_subdirectory(./att-client-snp) diff --git a/applications/test/att-client-sgx/main.cpp b/applications/test/att-client-sgx/main.cpp index 8a9343c..9831f99 100644 --- a/applications/test/att-client-sgx/main.cpp +++ b/applications/test/att-client-sgx/main.cpp @@ -1,6 +1,6 @@ -#include "abe4.h" -#include "attestation/attestation.h" -#include "jwt.h" +#include "accless/abe4/abe4.h" +#include "accless/attestation/attestation.h" +#include "accless/jwt/jwt.h" #include diff --git a/applications/test/att-client-snp/main.cpp b/applications/test/att-client-snp/main.cpp index 494af5b..9f9a9be 100644 --- a/applications/test/att-client-snp/main.cpp +++ b/applications/test/att-client-snp/main.cpp @@ -1,6 +1,6 @@ -#include "abe4.h" -#include "attestation/attestation.h" -#include "jwt.h" +#include "accless/abe4/abe4.h" +#include "accless/attestation/attestation.h" +#include "accless/jwt/jwt.h" #include From c00830c264bd74a543628db7b462125c85372aa5 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Fri, 21 Nov 2025 16:07:12 +0000 Subject: [PATCH 02/52] [accli] E: Clarify ApplicationsCommand Help --- accli/src/main.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/accli/src/main.rs b/accli/src/main.rs index a5bb16f..cc118f9 100644 --- a/accli/src/main.rs +++ b/accli/src/main.rs @@ -292,10 +292,13 @@ enum AcclessCommand { enum ApplicationsCommand { /// Build the Accless applications Build { + /// Force a clean build. #[arg(long)] clean: bool, + /// Force a debug build. #[arg(long)] debug: bool, + /// Path to the attestation service's public certificate PEM file. #[arg(long)] cert_path: Option, }, From ad4f89f4807419951b1581f1efca973e39b84324 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Fri, 21 Nov 2025 17:15:16 +0000 Subject: [PATCH 03/52] [scripts] E: Remove Obsolete Scripts --- scripts/build_cli.sh | 23 ----------------- scripts/patch_jwt_cert.sh | 52 --------------------------------------- 2 files changed, 75 deletions(-) delete mode 100755 scripts/build_cli.sh delete mode 100755 scripts/patch_jwt_cert.sh diff --git a/scripts/build_cli.sh b/scripts/build_cli.sh deleted file mode 100755 index 7fddf54..0000000 --- a/scripts/build_cli.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]:-${(%):-%x}}" )" >/dev/null 2>&1 && pwd )" -PROJ_ROOT="${THIS_DIR}/.." - -pushd ${PROJ_ROOT}>>/dev/null - -# ---------------------------- -# Environment vars -# ---------------------------- - -export VERSION=$(cat ${PROJ_ROOT}/VERSION) - -docker run \ - --rm -it \ - --name tless-build \ - --net host \ - -v ${PROJ_ROOT}/workflows:/code/faasm-examples/workflows \ - -w /code/faasm-examples \ - ghcr.io/coco-serverless/tless-experiments:${VERSION} \ - bash - -popd >> /dev/null diff --git a/scripts/patch_jwt_cert.sh b/scripts/patch_jwt_cert.sh deleted file mode 100755 index e82643a..0000000 --- a/scripts/patch_jwt_cert.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash - -# This script patches the known-good X509 certificate in the JWT parsing -# library after we deploy one instance of the attribute-providing-service. -# The service's certificate depends on the IP of where it is deployed, and -# it must be hard-coded inside the function code (for correct measurement). -# This file patches the source code once we have the deployed the service. - -set -euo pipefail - -# Get directories -THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]:-${(%):-%x}}" )" >/dev/null 2>&1 && pwd )" -PROJ_ROOT="${THIS_DIR}/.." - -# Define file paths -CERT_FILE="${PROJ_ROOT}/attestation-service/certs/cert.pem" -RUST_FILE="${PROJ_ROOT}/accless/libs/jwt/src/lib.rs" - -# Define markers -START_MARKER="// BEGIN: AUTO-INJECTED CERT" -END_MARKER="// END: AUTO-INJECTED CERT" - -if [[ ! -f "${CERT_FILE}" ]]; then - echo "accless: patch: error: Certificate file not found at ${CERT_FILE}" - echo "accless: patch: please run 'attestation-service/bin/gen_keys.sh' first." - exit 1 -fi - -if [[ ! -f "${RUST_FILE}" ]]; then - echo "accless: patch: error: JWT library file not found at ${RUST_FILE}" - exit 1 -fi - -echo "accless: patch: Reading new certificate from ${CERT_FILE}" - -# Read the certificate and format it as a Rust raw string literal -# We use awk to wrap the file content in `r#"` and `"#,\n` -NEW_CERT_BLOCK=$(awk 'BEGIN {print "r#\""} {print} END {print "\"#,"}' "${CERT_FILE}") - -# Use sed to replace the block between the markers -# This command finds the block, including the marker lines, and replaces it -# with the markers *plus* the new certificate block. -sed -i.bak "/${START_MARKER}/,/${END_MARKER}/c\\ -${START_MARKER}\ -${NEW_CERT_BLOCK}\ -${END_MARKER}\ -" "${RUST_FILE}" - -# Remove the backup file created by sed -rm -f "${RUST_FILE}.bak" - -echo "accless: patch: Successfully patched ${RUST_FILE} with new certificate." From f1a5ffa95c41909aa79e619a2fede687c4e573bb Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Sat, 22 Nov 2025 11:55:13 +0000 Subject: [PATCH 04/52] [accli] E: Add AttestationService Task --- accli/src/main.rs | 66 +++++++++++++++++- accli/src/tasks/attestation_service.rs | 95 ++++++++++++++++++++++++++ accli/src/tasks/mod.rs | 1 + 3 files changed, 161 insertions(+), 1 deletion(-) create mode 100644 accli/src/tasks/attestation_service.rs diff --git a/accli/src/main.rs b/accli/src/main.rs index cc118f9..3a4a8a1 100644 --- a/accli/src/main.rs +++ b/accli/src/main.rs @@ -3,6 +3,7 @@ use crate::{ tasks::{ accless::Accless, applications::Applications, + attestation_service::AttestationService, azure::Azure, dev::Dev, docker::{Docker, DockerContainer}, @@ -11,7 +12,7 @@ use crate::{ }, }; use clap::{Parser, Subcommand}; -use std::{collections::HashMap, process}; +use std::{collections::HashMap, path::PathBuf, process}; pub mod attestation_service; pub mod env; @@ -64,6 +65,44 @@ enum Command { #[command(subcommand)] s3_command: S3Command, }, + /// Build and run the attestation service + AttestationService { + #[command(subcommand)] + attestation_service_command: AttestationServiceCommand, + }, +} + +#[derive(Debug, Subcommand)] +enum AttestationServiceCommand { + /// Build the attestation service + Build {}, + /// Run the attestation service + Run { + /// Directory where to look-for and store TLS certificates. + #[arg(long)] + certs_dir: Option, + /// Port to bind the server to. + #[arg(long)] + port: Option, + /// URL to fetch SGX platform collateral information. + #[arg(long)] + sgx_pccs_url: Option, + /// Whether to overwrite the existing TLS certificates (if any). + #[arg(long)] + force_clean_certs: bool, + /// Run the attestation service in mock mode, skipping quote + /// verification. + #[arg(long, default_value_t = false)] + mock: bool, + }, + Health { + /// URL of the attestation service + #[arg(long)] + url: Option, + /// Path to the attestation service's public certificate PEM file + #[arg(long)] + cert_path: Option, + }, } #[derive(Debug, Subcommand)] @@ -760,6 +799,31 @@ async fn main() -> anyhow::Result<()> { } }, }, + Command::AttestationService { + attestation_service_command, + } => match attestation_service_command { + AttestationServiceCommand::Build {} => { + AttestationService::build()?; + } + AttestationServiceCommand::Run { + certs_dir, + port, + sgx_pccs_url, + force_clean_certs, + mock, + } => { + AttestationService::run( + certs_dir.as_deref(), + *port, + sgx_pccs_url.as_deref(), + *force_clean_certs, + *mock, + )?; + } + AttestationServiceCommand::Health { url, cert_path } => { + AttestationService::health(url.clone(), cert_path.clone()).await?; + } + }, } Ok(()) diff --git a/accli/src/tasks/attestation_service.rs b/accli/src/tasks/attestation_service.rs new file mode 100644 index 0000000..9d17c7b --- /dev/null +++ b/accli/src/tasks/attestation_service.rs @@ -0,0 +1,95 @@ +use crate::env::Env; +use anyhow::Result; +use log::info; +use reqwest; +use std::{ + fs, + process::{Command, Stdio}, +}; + +pub struct AttestationService; + +impl AttestationService { + pub fn build() -> Result<()> { + let mut cmd = Command::new("cargo"); + cmd.arg("build") + .arg("-p") + .arg("attestation-service") + .arg("--release"); + let status = cmd.status()?; + if !status.success() { + anyhow::bail!("Failed to build attestation service"); + } + Ok(()) + } + + pub fn run( + certs_dir: Option<&std::path::Path>, + port: Option, + sgx_pccs_url: Option<&std::path::Path>, + force_clean_certs: bool, + mock: bool, + ) -> Result<()> { + let mut cmd = Command::new( + Env::proj_root() + .join("target") + .join("release") + .join("attestation-service"), + ); + if let Some(certs_dir) = certs_dir { + cmd.arg("--certs-dir").arg(certs_dir); + } + if let Some(port) = port { + cmd.arg("--port").arg(port.to_string()); + } + if let Some(sgx_pccs_url) = sgx_pccs_url { + cmd.arg("--sgx-pccs-url").arg(sgx_pccs_url); + } + if force_clean_certs { + cmd.arg("--force-clean-certs"); + } + if mock { + cmd.arg("--mock"); + } + let status = cmd + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .status()?; + if !status.success() { + anyhow::bail!("Failed to run attestation service"); + } + Ok(()) + } + + pub async fn health(url: Option, cert_path: Option) -> Result<()> { + let url = url.or_else(|| std::env::var("ACCLESS_AS_URL").ok()); + let cert_path = cert_path.or_else(|| { + std::env::var("ACCLESS_AS_CERT_PATH") + .ok() + .map(std::path::PathBuf::from) + }); + + let url = match url { + Some(url) => url, + None => { + anyhow::bail!("Attestation service URL not provided. Set --url or ACCLESS_AS_URL") + } + }; + + let client = match cert_path { + Some(cert_path) => { + let cert = fs::read(cert_path)?; + let cert = reqwest::Certificate::from_pem(&cert)?; + reqwest::Client::builder() + .add_root_certificate(cert) + .build() + } + None => reqwest::Client::builder().build(), + }?; + + let response = client.get(format!("{}/health", url)).send().await?; + info!("Health check response: {}", response.text().await?); + + Ok(()) + } +} diff --git a/accli/src/tasks/mod.rs b/accli/src/tasks/mod.rs index a692518..ef7bf82 100644 --- a/accli/src/tasks/mod.rs +++ b/accli/src/tasks/mod.rs @@ -1,5 +1,6 @@ pub mod accless; pub mod applications; +pub mod attestation_service; pub mod azure; pub mod dev; pub mod docker; From d1326602fd834149d44bb1409ecd6ff53192a966 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Sat, 22 Nov 2025 13:44:32 +0000 Subject: [PATCH 05/52] [scripts] E: Add Scripts To Spawn SNP cVM --- scripts/apt.sh | 4 +- scripts/common/logging.sh | 97 +++++++++++++ scripts/common/utils.sh | 41 ++++++ scripts/snp/.gitignore | 1 + scripts/snp/cloud-init/meta-data.in | 2 + scripts/snp/cloud-init/user-data.in | 43 ++++++ scripts/snp/run.sh | 55 ++++++++ scripts/snp/setup.sh | 211 ++++++++++++++++++++++++++++ scripts/snp/versions.sh | 4 + 9 files changed, 457 insertions(+), 1 deletion(-) create mode 100644 scripts/common/logging.sh create mode 100644 scripts/common/utils.sh create mode 100644 scripts/snp/.gitignore create mode 100644 scripts/snp/cloud-init/meta-data.in create mode 100644 scripts/snp/cloud-init/user-data.in create mode 100755 scripts/snp/run.sh create mode 100755 scripts/snp/setup.sh create mode 100644 scripts/snp/versions.sh diff --git a/scripts/apt.sh b/scripts/apt.sh index e036a85..b07e4d8 100755 --- a/scripts/apt.sh +++ b/scripts/apt.sh @@ -4,4 +4,6 @@ sudo apt install -y \ clang-format \ libfontconfig1-dev \ libssl-dev \ - pkg-config > /dev/null 2>&1 + ovmf \ + pkg-config \ + qemu-system-x86 > /dev/null 2>&1 diff --git a/scripts/common/logging.sh b/scripts/common/logging.sh new file mode 100644 index 0000000..cc3f331 --- /dev/null +++ b/scripts/common/logging.sh @@ -0,0 +1,97 @@ +#!/bin/bash + +# +# Logging functions. +# + +#=================================================================================================== +# Include Guard +#=================================================================================================== + +# Skip this file if already included. +if [[ -n "${__LOGGING_SH_INCLUDED:-}" ]]; then + return +fi +readonly __LOGGING_SH_INCLUDED=1 + +#=================================================================================================== +# Constants +#=================================================================================================== + +# Colors +readonly RED='\033[0;31m' # Red +readonly GREEN='\033[0;32m' # Green +readonly YELLOW='\033[0;33m' # Yellow +readonly NC='\033[0m' # No Color + +#================================================================================================== +# Functions +#================================================================================================== + +# +# Description +# +# Prints an error message on stderr. +# +# Arguments +# +# $1 - The error message to print. +# +# Usage Example +# +# print_error "Print an error message." +# +print_error() { + echo -e "${RED}[ERROR] ${1}${NC}" >&2 +} + +# +# Description +# +# Prints a success message on stdout. +# +# Arguments +# +# $1 - The success message to print. +# +# Usage Example +# +# print_success "Print a success message." +# +print_success() { + echo -e "${GREEN}[SUCCESS] ${1}${NC}" +} + +# +# Description +# +# Prints a message on stdout. +# +# Arguments +# +# $1 - The message to print. +# +# Usage Example +# +# print_info "Print an informational message." +# +print_info() { + echo -e "[INFO] ${1}" +} + +# +# Description +# +# Prints a warning message on stderr. +# +# Arguments +# +# $1 - The warning message to print. +# +# Usage Example +# +# print_warning "Print a warning message." +# +print_warning() { + echo -e "${YELLOW}[WARN] ${1}${NC}" >&2 +} diff --git a/scripts/common/utils.sh b/scripts/common/utils.sh new file mode 100644 index 0000000..9b81d4e --- /dev/null +++ b/scripts/common/utils.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# Copyright(c) The Maintainers of Nanvix. +# Licensed under the MIT License. + +# +# Utility functions. +# + +#=================================================================================================== +# Include Guard +#=================================================================================================== + +# Skip this file if already included. +if [[ -n "${__UTILS_SH_INCLUDED:-}" ]]; then + return +fi +readonly __UTILS_SH_INCLUDED=1 + +#================================================================================================== +# Imports +#================================================================================================== + +source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/logging.sh" + +#================================================================================================== +# Functions +#================================================================================================== + +# +# Checks if the --clean flag was passed. +# Returns 0 (true) if the flag is present, 1 (false) otherwise. +# +has_clean_flag() { + for arg in "$@"; do + if [[ "$arg" == "--clean" ]]; then + return 0 # 0 means "true" (success) + fi + done + return 1 # 1 means "false" (failure) +} diff --git a/scripts/snp/.gitignore b/scripts/snp/.gitignore new file mode 100644 index 0000000..53752db --- /dev/null +++ b/scripts/snp/.gitignore @@ -0,0 +1 @@ +output diff --git a/scripts/snp/cloud-init/meta-data.in b/scripts/snp/cloud-init/meta-data.in new file mode 100644 index 0000000..5e212f0 --- /dev/null +++ b/scripts/snp/cloud-init/meta-data.in @@ -0,0 +1,2 @@ +instance-id: ${INSTANCE_ID} +local-hostname: accless-snp diff --git a/scripts/snp/cloud-init/user-data.in b/scripts/snp/cloud-init/user-data.in new file mode 100644 index 0000000..905a95e --- /dev/null +++ b/scripts/snp/cloud-init/user-data.in @@ -0,0 +1,43 @@ +#cloud-config + +users: + - name: ubuntu + gecos: Ubuntu + sudo: ALL=(ALL) NOPASSWD:ALL + groups: [sudo, docker] + shell: /bin/bash + ssh_authorized_keys: + - "${SSH_PUB_KEY}" + +growpart: + mode: auto + devices: ['/'] + ignore_growroot_disabled: false + +package_update: true +packages: + - cargo + - docker.io + - git + - linux-generic + - rustc + +runcmd: + # Make sure ubuntu is in docker group (in case group created after user). + - [ sh, -c, 'usermod -aG docker ubuntu || true' ] + # Enable and start docker. + - [ systemctl, enable, --now, docker ] + # Enable and start sshd. + - [ systemctl, enable, --now, ssh.service ] + # Load SEV guest driver so /dev/sev-guest exists. + - [ sh, -c, 'modprobe sev-guest || echo "modprobe sev-guest failed (maybe built-in?)"' ] + # Change password. + - [ sh, -c, 'echo "ubuntu:ubuntu" | chpasswd' ] + # Make sure home is owned by ubuntu> + - [ chown, "-R", "ubuntu:ubuntu", "/home/ubuntu" ] + # Clone Accless repo. + - [ sudo, "-u", "ubuntu", "bash", "-lc", "cd /home/ubuntu && git clone https://github.com/faasm/accless.git accless" ] + # Quick debug markers in console. + - [ sh, -c, 'echo "[cloud-init] Docker + chpasswd + sev-guest setup done"' ] + +final_message: "Accless SNP test instance v2 is ready." diff --git a/scripts/snp/run.sh b/scripts/snp/run.sh new file mode 100755 index 0000000..3120b6b --- /dev/null +++ b/scripts/snp/run.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +set -euo pipefail + +THIS_DIR="$(dirname "$(realpath "$0")")" +SCRIPTS_DIR="${THIS_DIR}/.." +OUTPUT_DIR="${THIS_DIR}/output" + +source "${SCRIPTS_DIR}/common/utils.sh" +source "${THIS_DIR}/versions.sh" + +# +# Helper method to get the C-bit position directly from hardware. +# +get_cbitpos() { + modprobe cpuid + local ebx=$(sudo dd if=/dev/cpu/0/cpuid ibs=16 count=32 skip=134217728 2> /dev/null | tail -c 16 | od -An -t u4 -j 4 -N 4 | sed -re 's|^ *||') + local cbitpos=$((ebx & 0x3f)) + echo $cbitpos +} + +# +# Run an SNP guest using QEMU. +# +run_qemu() { + local qemu_dir="${OUTPUT_DIR}/qemu/qemu-${QEMU_VERSION}" + local qemu="${qemu_dir}/build/qemu-system-x86_64" + local qemu_bios_dir="${qemu_dir}/pc-bios" + + local kernel="${OUTPUT_DIR}/vmlinuz-noble" + local ovmf="${OUTPUT_DIR}/ovmf/ovmf-${OVMF_VERSION}/Build/AmdSev/RELEASE_GCC5/FV/OVMF.fd" + local disk_image="${OUTPUT_DIR}/disk.img" + local seed_image="${OUTPUT_DIR}/seed.img" + local cbitpos=$(get_cbitpos) + + # Can SSH into the VM witih: + # ssh -p 2222 -i ${OUTPUT_DIR}/snp-key ubuntu@localhost + ${qemu} \ + -L "${qemu_bios_dir}" \ + -enable-kvm \ + -nographic \ + -machine q35,confidential-guest-support=sev0,vmport=off \ + -cpu EPYC-v4 \ + -smp 6 -m 6G \ + -bios ${ovmf} \ + -object memory-backend-memfd,id=ram1,size=6G,share=true,prealloc=false \ + -machine memory-backend=ram1 \ + -object sev-snp-guest,id=sev0,cbitpos=${cbitpos},reduced-phys-bits=1 \ + -drive "if=virtio,format=qcow2,file=${disk_image}" \ + -drive "if=virtio,format=raw,file=${seed_image}" \ + -netdev user,id=net0,hostfwd=tcp::2222-:22 \ + -device e1000,netdev=net0 +} + +run_qemu diff --git a/scripts/snp/setup.sh b/scripts/snp/setup.sh new file mode 100755 index 0000000..f0117cd --- /dev/null +++ b/scripts/snp/setup.sh @@ -0,0 +1,211 @@ +#!/bin/bash + +set -euo pipefail + +THIS_DIR="$(dirname "$(realpath "$0")")" +SCRIPTS_DIR="${THIS_DIR}/.." +OUTPUT_DIR="${THIS_DIR}/output" + +source "${SCRIPTS_DIR}/common/utils.sh" +source "${THIS_DIR}/versions.sh" + +clean() { + print_info "Cleaning and re-creating ${OUTPUT_DIR} directory..." + rm -rf ${OUTPUT_DIR} + mkdir -p ${OUTPUT_DIR} +} + +# TODO: check host kernel + +# +# Fetch the linux kernel image. +# +install_apt_deps() { + print_info "Installing APT dependencies..." + sudo apt install -y cloud-utils ovmf > /dev/null 2>&1 +} + +# +# Build up-to-date QEMU (need >= 9.x). +# +build_qemu() { + print_info "Building and installing QEMU (v${QEMU_VERSION}) from source (will take a minute)..." + local qemu_out_dir="${OUTPUT_DIR}/qemu" + mkdir -p ${qemu_out_dir} + pushd ${qemu_out_dir} >> /dev/null + + wget https://download.qemu.org/qemu-${QEMU_VERSION}.tar.xz > /dev/null 2>&1 + tar xvJf qemu-${QEMU_VERSION}.tar.xz > /dev/null 2>&1 + pushd qemu-${QEMU_VERSION} >> /dev/null + ./configure --enable-slirp > /dev/null 2>&1 + make -j $(nproc) > /dev/null 2>&1 + popd >> /dev/null + + popd >> /dev/null + print_success "Successfully built QEMU (v${QEMU_VERSION})!" +} + +# +# Build up-to-date OVMF version. +# +build_ovmf() { + print_info "Building and installing OVMF (${OVMF_VERSION}) from source..." + local ovmf_out_dir="${OUTPUT_DIR}/ovmf" + mkdir -p ${ovmf_out_dir} + pushd ${ovmf_out_dir} >> /dev/null + + if [ ! -d "ovmf-${OVMF_VERSION}" ]; then + git clone -b ${OVMF_VERSION} https://github.com/tianocore/edk2.git ovmf-${OVMF_VERSION} > /dev/null 2>&1 + fi + + pushd ovmf-${OVMF_VERSION} >> /dev/null + git submodule update --init --recursive > /dev/null 2>&1 + make -C BaseTools clean > /dev/null 2>&1 + make -C BaseTools -j $(nproc) > /dev/null 2>&1 + . ./edksetup.sh --reconfig + build -a X64 -b RELEASE -t GCC5 -p OvmfPkg/OvmfPkgX64.dsc > /dev/null 2>&1 + touch OvmfPkg/AmdSev/Grub/grub.efi > /dev/null 2>&1 + build -a X64 -b RELEASE -t GCC5 -p OvmfPkg/AmdSev/AmdSevX64.dsc > /dev/null 2>&1 + popd >> /dev/null + + popd >> /dev/null + print_success "Successfully built OVMF (${OVMF_VERSION})!" +} + +# +# Fetch the linux kernel image. +# +fetch_kernel() { + print_info "Fetching Linux kernel..." + wget \ + https://cloud-images.ubuntu.com/noble/20251113/unpacked/noble-server-cloudimg-amd64-vmlinuz-generic \ + -O ${OUTPUT_DIR}/vmlinuz-noble > /dev/null 2>&1 + if [ $? -eq 0 ]; then + print_success "Linux kernel fetched successfully." + else + print_error "Failed to fetch Linux kernel." + exit 1 + fi +} + +# +# Fetch the linux kernel image. +# +fetch_disk_image() { + print_info "Fetching cloud-init disk image..." + wget \ + https://cloud-images.ubuntu.com/noble/20251113/noble-server-cloudimg-amd64.img \ + -O ${OUTPUT_DIR}/disk.img > /dev/null 2>&1 + if [ $? -eq 0 ]; then + print_success "cloud-init disk image fetched successfully." + else + print_error "Failed to fetch cloud-init disk image." + exit 1 + fi + + local qemu_img="${OUTPUT_DIR}/qemu/qemu-${QEMU_VERSION}/build/qemu-img" + ${qemu_img} resize "${OUTPUT_DIR}/disk.img" +20G > /dev/null 2>&1 + print_success "cloud-init disk image resized successfully." +} + +# +# Generate ephemeral keypair for VM. +# +generate_ephemeral_keys() { + print_info "Generating ephemeral keypair..." + ssh-keygen -q -t ed25519 -N "" -f ${OUTPUT_DIR}/snp-key <<< y >/dev/null 2>&1 + print_info "Keypair generated succesfully!" +} + +# +# Prepare the cloudinit overlay disk image. +# +prepare_cloudinit_image() { + print_info "Preparing cloud-init overlay image..." + + local in_dir="${THIS_DIR}/cloud-init" + local out_dir="${OUTPUT_DIR}/cloud-init" + + mkdir -p ${out_dir} + INSTANCE_ID="accless-snp-$(date +%s)" envsubst '${INSTANCE_ID}' \ + < ${in_dir}/meta-data.in > ${out_dir}/meta-data + SSH_PUB_KEY=$(cat "${OUTPUT_DIR}/snp-key.pub") envsubst '${SSH_PUB_KEY}' \ + < ${in_dir}/user-data.in > ${out_dir}/user-data + + cloud-localds "${OUTPUT_DIR}/seed.img" "${out_dir}/user-data" "${out_dir}/meta-data" + + print_success "cloud-init overlay prepared successfully!" +} + +usage() { + print_info "Usage: $0 [--clean] [--component ]" + exit 1 +} + +main() { + local component="" + + while [[ "$#" -gt 0 ]]; do + case $1 in + --clean) + clean + shift + ;; + --component) + if [[ -z "$2" || "$2" == --* ]]; then + echo "Error: --component requires a value." + usage + fi + component="$2" + shift 2 + ;; + -h | --help) + usage + ;; + *) + print_error "Unknown option: $1" + usage + ;; + esac + done + + if [[ -n "$component" ]]; then + case "$component" in + apt) + install_apt_deps + ;; + qemu) + build_qemu + ;; + ovmf) + build_ovmf + ;; + kernel) + fetch_kernel + ;; + disk) + fetch_disk_image + ;; + keys) + generate_ephemeral_keys + ;; + cloudinit) + prepare_cloudinit_image + ;; + *) + print_error "Error: Invalid component '$component'" + usage + ;; + esac + else + install_apt_deps + build_qemu + build_ovmf + fetch_kernel + fetch_disk_image + generate_ephemeral_keys + prepare_cloudinit_image + fi +} + +main "$@" diff --git a/scripts/snp/versions.sh b/scripts/snp/versions.sh new file mode 100644 index 0000000..d6cbd54 --- /dev/null +++ b/scripts/snp/versions.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +export OVMF_VERSION="edk2-stable202511" +export QEMU_VERSION="10.1.2" From 568b2d13f7af24d99f35f79ea7ce086ca6d26dca Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Sat, 22 Nov 2025 13:57:32 +0000 Subject: [PATCH 06/52] [accli] B: Ignore Paths In Dev --- accli/src/tasks/applications.rs | 30 ++++++++++++++------- accli/src/tasks/dev.rs | 11 ++++++-- accli/src/tasks/docker.rs | 48 +++++++++++++++++++++++++++++++++ 3 files changed, 77 insertions(+), 12 deletions(-) diff --git a/accli/src/tasks/applications.rs b/accli/src/tasks/applications.rs index cfc6b54..cbd6bb6 100644 --- a/accli/src/tasks/applications.rs +++ b/accli/src/tasks/applications.rs @@ -18,18 +18,28 @@ impl Applications { if debug { cmd.push("--debug".to_string()); } - if let Some(cert_path) = cert_path { + if let Some(cert_path_str) = cert_path { + let cert_path = Path::new(cert_path_str); + if !cert_path.exists() { + anyhow::bail!("Certificate path does not exist: {}", cert_path.display()); + } + if !cert_path.is_file() { + anyhow::bail!("Certificate path is not a file: {}", cert_path.display()); + } + let docker_cert_path = Docker::get_docker_path(cert_path)?; cmd.push("--cert-path".to_string()); - cmd.push(cert_path.to_string()); + let docker_cert_path_str = docker_cert_path.to_str().ok_or_else(|| { + anyhow::anyhow!( + "Docker path for certificate is not valid UTF-8: {}", + docker_cert_path.display() + ) + })?; + cmd.push(docker_cert_path_str.to_string()); } let workdir = Path::new(DOCKER_ACCLESS_CODE_MOUNT_DIR).join("applications"); - Docker::run( - &cmd, - true, - Some(workdir.to_str().unwrap()), - &[], - false, - capture_output, - ) + let workdir_str = workdir.to_str().ok_or_else(|| { + anyhow::anyhow!("Workdir path is not valid UTF-8: {}", workdir.display()) + })?; + Docker::run(&cmd, true, Some(workdir_str), &[], false, capture_output) } } diff --git a/accli/src/tasks/dev.rs b/accli/src/tasks/dev.rs index 7905432..bb5221d 100644 --- a/accli/src/tasks/dev.rs +++ b/accli/src/tasks/dev.rs @@ -209,7 +209,14 @@ impl Dev { } fn is_excluded(entry: &walkdir::DirEntry) -> bool { - let excluded_dirs = ["build-wasm", "build-native", "target", "venv", "venv-bm"]; + let excluded_dirs = [ + "build-wasm", + "build-native", + "output", + "target", + "venv", + "venv-bm", + ]; entry.file_type().is_dir() && entry .file_name() @@ -218,7 +225,7 @@ impl Dev { .unwrap_or(false) } - for entry in walkdir::WalkDir::new(".") + for entry in walkdir::WalkDir::new(Env::proj_root()) .into_iter() .filter_entry(|e| !is_excluded(e)) .filter_map(Result::ok) diff --git a/accli/src/tasks/docker.rs b/accli/src/tasks/docker.rs index 75afbb2..7617bcf 100644 --- a/accli/src/tasks/docker.rs +++ b/accli/src/tasks/docker.rs @@ -3,6 +3,7 @@ use clap::ValueEnum; use log::error; use std::{ fmt, + path::{Path, PathBuf}, process::{Command, Stdio}, str::FromStr, }; @@ -50,6 +51,53 @@ pub const DOCKER_ACCLESS_CODE_MOUNT_DIR: &str = "/code/accless"; impl Docker { const ACCLESS_DEV_CONTAINER_NAME: &'static str = "accless-dev"; + /// # Description + /// + /// This function takes a path from the host filesystem and maps it to the + /// corresponding path inside the Docker container. + /// It first converts the `host_path` to an absolute path and canonicalizes + /// it. This process also verifies that the path exists. + /// Then, it checks if the path is within the project's root directory. + /// If it is, it strips the project root prefix and prepends the Docker + /// mount directory path (`/code/accless`). + /// + /// # Arguments + /// + /// * `host_path` - A reference to a `Path` on the host filesystem. It can + /// be either an absolute path or a path relative to the current working + /// directory. + /// + /// # Returns + /// + /// A `anyhow::Result` which is: + /// - `Ok(PathBuf)`: The mapped path inside the Docker container. + /// - `Err(anyhow::Error)`: An error if: + /// - The path does not exist or cannot be canonicalized. + /// - The path is outside the project's root directory. + pub fn get_docker_path(host_path: &Path) -> anyhow::Result { + let absolute_host_path = if host_path.is_absolute() { + host_path.to_path_buf() + } else { + std::env::current_dir()?.join(host_path) + }; + let absolute_host_path = absolute_host_path.canonicalize().map_err(|e| { + anyhow::anyhow!("Error canonicalizing path {}: {}", host_path.display(), e) + })?; + + let proj_root = Env::proj_root(); + if absolute_host_path.starts_with(&proj_root) { + let relative_path = absolute_host_path.strip_prefix(&proj_root).unwrap(); + let docker_path = Path::new(DOCKER_ACCLESS_CODE_MOUNT_DIR).join(relative_path); + Ok(docker_path) + } else { + anyhow::bail!( + "Path {} is outside the project root directory {}", + absolute_host_path.display(), + proj_root.display() + ); + } + } + pub fn get_docker_tag(ctr: &DockerContainer) -> String { // Prepare image tag let version = match Env::get_version() { From fb7c36687f9eb2289dd320f109360af0ae900a0d Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Sat, 22 Nov 2025 16:10:47 +0000 Subject: [PATCH 07/52] [scripts] E: Refine SNP SetUp Scripts --- scripts/apt.sh | 5 ++--- scripts/snp/cloud-init/user-data.in | 34 +++++++++++++++++++++++++---- scripts/snp/run.sh | 6 ++++- scripts/snp/setup.sh | 19 ++++++++++++++-- scripts/snp/versions.sh | 2 ++ 5 files changed, 56 insertions(+), 10 deletions(-) diff --git a/scripts/apt.sh b/scripts/apt.sh index b07e4d8..705c540 100755 --- a/scripts/apt.sh +++ b/scripts/apt.sh @@ -1,9 +1,8 @@ #!/bin/bash sudo apt install -y \ + build-essential \ clang-format \ libfontconfig1-dev \ libssl-dev \ - ovmf \ - pkg-config \ - qemu-system-x86 > /dev/null 2>&1 + pkg-config > /dev/null 2>&1 diff --git a/scripts/snp/cloud-init/user-data.in b/scripts/snp/cloud-init/user-data.in index 905a95e..632790b 100644 --- a/scripts/snp/cloud-init/user-data.in +++ b/scripts/snp/cloud-init/user-data.in @@ -16,11 +16,10 @@ growpart: package_update: true packages: - - cargo - docker.io - git - - linux-generic - - rustc + - linux-modules-extra-${GUEST_KERNEL_VERSION} + - python3-venv runcmd: # Make sure ubuntu is in docker group (in case group created after user). @@ -33,10 +32,37 @@ runcmd: - [ sh, -c, 'modprobe sev-guest || echo "modprobe sev-guest failed (maybe built-in?)"' ] # Change password. - [ sh, -c, 'echo "ubuntu:ubuntu" | chpasswd' ] + - [ bash, -c, ' + if [ "$(id -u ubuntu)" -eq 1000 ]; then + echo "[cloud-init] Changing ubuntu UID/GID from 1000 to 2000"; + + # Change group first + groupmod -g 2000 ubuntu; + + # Change user UID + usermod -u 2000 ubuntu; + + # Fix ownership of files that still belong to uid/gid 1000 + # Restrict to typical places to avoid scanning the whole fs if you prefer + for path in /home /var /etc; do + find "$path" -xdev -uid 1000 -exec chown -h 2000:2000 {} \; || true + find "$path" -xdev -gid 1000 -exec chgrp -h 2000 {} \; || true + done + + echo "[cloud-init] ubuntu is now: $(id ubuntu)"; + else + echo "[cloud-init] ubuntu UID is $(id -u ubuntu), not 1000; skipping UID change."; + fi + ' ] # Make sure home is owned by ubuntu> - [ chown, "-R", "ubuntu:ubuntu", "/home/ubuntu" ] + # Install rust. + - [ sudo, "-u", "ubuntu", "bash", "-lc", "curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y" ] # Clone Accless repo. - - [ sudo, "-u", "ubuntu", "bash", "-lc", "cd /home/ubuntu && git clone https://github.com/faasm/accless.git accless" ] + # FIXME: remove branch name. + - [ sudo, "-u", "ubuntu", "bash", "-lc", "cd /home/ubuntu && git clone -b feature-escrow-func https://github.com/faasm/tless.git accless" ] + # Fetch Accless docker image. + - [ sudo, "-u", "ubuntu", "bash", "-lc", "docker pull ghcr.io/faasm/accless-experiments:${ACCLESS_VERSION}" ] # Quick debug markers in console. - [ sh, -c, 'echo "[cloud-init] Docker + chpasswd + sev-guest setup done"' ] diff --git a/scripts/snp/run.sh b/scripts/snp/run.sh index 3120b6b..d19292a 100755 --- a/scripts/snp/run.sh +++ b/scripts/snp/run.sh @@ -28,6 +28,7 @@ run_qemu() { local qemu_bios_dir="${qemu_dir}/pc-bios" local kernel="${OUTPUT_DIR}/vmlinuz-noble" + local initrd="${OUTPUT_DIR}/initrd-noble" local ovmf="${OUTPUT_DIR}/ovmf/ovmf-${OVMF_VERSION}/Build/AmdSev/RELEASE_GCC5/FV/OVMF.fd" local disk_image="${OUTPUT_DIR}/disk.img" local seed_image="${OUTPUT_DIR}/seed.img" @@ -43,9 +44,12 @@ run_qemu() { -cpu EPYC-v4 \ -smp 6 -m 6G \ -bios ${ovmf} \ + -kernel ${kernel} \ + -append "root=/dev/vda1 console=ttyS0" \ + -initrd ${initrd} \ -object memory-backend-memfd,id=ram1,size=6G,share=true,prealloc=false \ -machine memory-backend=ram1 \ - -object sev-snp-guest,id=sev0,cbitpos=${cbitpos},reduced-phys-bits=1 \ + -object sev-snp-guest,id=sev0,cbitpos=${cbitpos},reduced-phys-bits=1,kernel-hashes=on \ -drive "if=virtio,format=qcow2,file=${disk_image}" \ -drive "if=virtio,format=raw,file=${seed_image}" \ -netdev user,id=net0,hostfwd=tcp::2222-:22 \ diff --git a/scripts/snp/setup.sh b/scripts/snp/setup.sh index f0117cd..b53a652 100755 --- a/scripts/snp/setup.sh +++ b/scripts/snp/setup.sh @@ -4,6 +4,7 @@ set -euo pipefail THIS_DIR="$(dirname "$(realpath "$0")")" SCRIPTS_DIR="${THIS_DIR}/.." +ROOT_DIR="${SCRIPTS_DIR}/.." OUTPUT_DIR="${THIS_DIR}/output" source "${SCRIPTS_DIR}/common/utils.sh" @@ -78,7 +79,7 @@ build_ovmf() { fetch_kernel() { print_info "Fetching Linux kernel..." wget \ - https://cloud-images.ubuntu.com/noble/20251113/unpacked/noble-server-cloudimg-amd64-vmlinuz-generic \ + https://cloud-images.ubuntu.com/noble/${UBUNTU_VERSION}/unpacked/noble-server-cloudimg-amd64-vmlinuz-generic \ -O ${OUTPUT_DIR}/vmlinuz-noble > /dev/null 2>&1 if [ $? -eq 0 ]; then print_success "Linux kernel fetched successfully." @@ -86,6 +87,17 @@ fetch_kernel() { print_error "Failed to fetch Linux kernel." exit 1 fi + + print_info "Fetching initrd..." + wget \ + https://cloud-images.ubuntu.com/noble/${UBUNTU_VERSION}/unpacked/noble-server-cloudimg-amd64-initrd-generic \ + -O ${OUTPUT_DIR}/initrd-noble > /dev/null 2>&1 + if [ $? -eq 0 ]; then + print_success "Kernel initrd fetched successfully." + else + print_error "Failed to fetch kernel initrd." + exit 1 + fi } # @@ -129,7 +141,10 @@ prepare_cloudinit_image() { mkdir -p ${out_dir} INSTANCE_ID="accless-snp-$(date +%s)" envsubst '${INSTANCE_ID}' \ < ${in_dir}/meta-data.in > ${out_dir}/meta-data - SSH_PUB_KEY=$(cat "${OUTPUT_DIR}/snp-key.pub") envsubst '${SSH_PUB_KEY}' \ + ACCLESS_VERSION=$(cat "${ROOT_DIR}/VERSION") \ + GUEST_KERNEL_VERSION=${GUEST_KERNEL_VERSION} \ + SSH_PUB_KEY=$(cat "${OUTPUT_DIR}/snp-key.pub") \ + envsubst '${ACCLESS_VERSION} ${GUEST_KERNEL_VERSION} ${SSH_PUB_KEY}' \ < ${in_dir}/user-data.in > ${out_dir}/user-data cloud-localds "${OUTPUT_DIR}/seed.img" "${out_dir}/user-data" "${out_dir}/meta-data" diff --git a/scripts/snp/versions.sh b/scripts/snp/versions.sh index d6cbd54..36cbc8b 100644 --- a/scripts/snp/versions.sh +++ b/scripts/snp/versions.sh @@ -1,4 +1,6 @@ #!/bin/bash +export GUEST_KERNEL_VERSION="6.8.0-87-generic" export OVMF_VERSION="edk2-stable202511" export QEMU_VERSION="10.1.2" +export UBUNTU_VERSION="20251113" From 3b0b05412eaffb9379ef499f7bed898fb1e13dbd Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Sat, 22 Nov 2025 16:11:05 +0000 Subject: [PATCH 08/52] [attestation-service] E: Log Node URL On Startup --- attestation-service/src/main.rs | 1 + attestation-service/src/tls.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/attestation-service/src/main.rs b/attestation-service/src/main.rs index 833e51e..ebe4d97 100644 --- a/attestation-service/src/main.rs +++ b/attestation-service/src/main.rs @@ -85,6 +85,7 @@ async fn main() -> Result<()> { let listener = TcpListener::bind(addr).await; info!("Accless attestation server running on https://{}", addr); + info!("External IP: {}", tls::get_node_url()?); loop { let (stream, _) = listener.as_ref().expect("error listening").accept().await?; let service = TowerToHyperService::new(app.clone()); diff --git a/attestation-service/src/tls.rs b/attestation-service/src/tls.rs index 52e3241..d7ee135 100644 --- a/attestation-service/src/tls.rs +++ b/attestation-service/src/tls.rs @@ -59,7 +59,7 @@ pub fn get_public_certificate_path(certs_dir: &Path) -> PathBuf { /// # Returns /// /// The external node IP as a string. -fn get_node_url() -> Result { +pub fn get_node_url() -> Result { let output = Command::new("ip") .args(["-o", "route", "get", "to", "8.8.8.8"]) .output()?; From a4f5be7d3a3ac56217d7ba74f2b256c0de13ece0 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Sat, 22 Nov 2025 16:11:27 +0000 Subject: [PATCH 09/52] [accli] E: Add --rebuild Flag To attestation-service run --- accli/src/main.rs | 5 +++++ accli/src/tasks/attestation_service.rs | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/accli/src/main.rs b/accli/src/main.rs index 3a4a8a1..33cca38 100644 --- a/accli/src/main.rs +++ b/accli/src/main.rs @@ -94,6 +94,9 @@ enum AttestationServiceCommand { /// verification. #[arg(long, default_value_t = false)] mock: bool, + /// Rebuild the attestation service before running. + #[arg(long, default_value_t = false)] + rebuild: bool, }, Health { /// URL of the attestation service @@ -811,6 +814,7 @@ async fn main() -> anyhow::Result<()> { sgx_pccs_url, force_clean_certs, mock, + rebuild, } => { AttestationService::run( certs_dir.as_deref(), @@ -818,6 +822,7 @@ async fn main() -> anyhow::Result<()> { sgx_pccs_url.as_deref(), *force_clean_certs, *mock, + *rebuild, )?; } AttestationServiceCommand::Health { url, cert_path } => { diff --git a/accli/src/tasks/attestation_service.rs b/accli/src/tasks/attestation_service.rs index 9d17c7b..7ddf35f 100644 --- a/accli/src/tasks/attestation_service.rs +++ b/accli/src/tasks/attestation_service.rs @@ -29,7 +29,12 @@ impl AttestationService { sgx_pccs_url: Option<&std::path::Path>, force_clean_certs: bool, mock: bool, + rebuild: bool, ) -> Result<()> { + if rebuild { + Self::build()?; + } + let mut cmd = Command::new( Env::proj_root() .join("target") From 1d429a33665588ae2d8567c167f1970ba9c55343 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Sat, 22 Nov 2025 16:30:54 +0000 Subject: [PATCH 10/52] [accli] E: Pass /dev/sev-guest To Docker If Available --- accli/src/tasks/docker.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/accli/src/tasks/docker.rs b/accli/src/tasks/docker.rs index 7617bcf..fb26d44 100644 --- a/accli/src/tasks/docker.rs +++ b/accli/src/tasks/docker.rs @@ -268,6 +268,10 @@ impl Docker { run_cmd.arg("--network").arg("host"); } + if Path::new("/dev/sev-guest").exists() { + run_cmd.arg("--device=/dev/sev-guest"); + } + if mount { run_cmd .arg("-v") From 23c96fbd76a9ad8ddfcf3a280997b0c64bb70af5 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Sat, 22 Nov 2025 16:40:18 +0000 Subject: [PATCH 11/52] [applications] E: Add Preliminary Escrow-Xput Function --- applications/CMakeLists.txt | 4 +- applications/functions/CMakeLists.txt | 1 + .../functions/escrow-xput/CMakeLists.txt | 10 + .../functions/escrow-xput/include/logger.h | 9 + .../functions/escrow-xput/include/utils.h | 5 + .../functions/escrow-xput/src/logger.cpp | 22 + .../functions/escrow-xput/src/main.cpp | 604 ++++++++++++++++++ .../functions/escrow-xput/src/utils.cpp | 14 + 8 files changed, 666 insertions(+), 3 deletions(-) create mode 100644 applications/functions/CMakeLists.txt create mode 100644 applications/functions/escrow-xput/CMakeLists.txt create mode 100644 applications/functions/escrow-xput/include/logger.h create mode 100644 applications/functions/escrow-xput/include/utils.h create mode 100644 applications/functions/escrow-xput/src/logger.cpp create mode 100644 applications/functions/escrow-xput/src/main.cpp create mode 100644 applications/functions/escrow-xput/src/utils.cpp diff --git a/applications/CMakeLists.txt b/applications/CMakeLists.txt index 18c146b..4d23bfb 100644 --- a/applications/CMakeLists.txt +++ b/applications/CMakeLists.txt @@ -31,11 +31,9 @@ set(ACCLESS_HEADERS ${ACCLESS_INCLUDE_PREFIX_DIR}) # Application subdirectories # ============================================================================= -# Test applications. if (NOT CMAKE_SYSTEM_NAME STREQUAL "WASI") + add_subdirectory(./functions) add_subdirectory(./test) endif () # To-Do: add workflows - -# To-Do: add functions diff --git a/applications/functions/CMakeLists.txt b/applications/functions/CMakeLists.txt new file mode 100644 index 0000000..71c5603 --- /dev/null +++ b/applications/functions/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(./escrow-xput) diff --git a/applications/functions/escrow-xput/CMakeLists.txt b/applications/functions/escrow-xput/CMakeLists.txt new file mode 100644 index 0000000..dccda16 --- /dev/null +++ b/applications/functions/escrow-xput/CMakeLists.txt @@ -0,0 +1,10 @@ +set(CMAKE_PROJECT_TARGET escrow-xput) + +add_executable(${CMAKE_PROJECT_TARGET} + src/main.cpp + # src/logger.cpp + # src/utils.cpp +) + +target_include_directories(${CMAKE_PROJECT_TARGET} PRIVATE ${ACCLESS_HEADERS}) +target_link_libraries(${CMAKE_PROJECT_TARGET} PRIVATE accless::accless) diff --git a/applications/functions/escrow-xput/include/logger.h b/applications/functions/escrow-xput/include/logger.h new file mode 100644 index 0000000..d6803f1 --- /dev/null +++ b/applications/functions/escrow-xput/include/logger.h @@ -0,0 +1,9 @@ +#pragma once + +#include + +class Logger : public attest::AttestationLogger { + public: + void Log(const char *log_tag, LogLevel level, const char *function, + const int line, const char *fmt, ...); +}; diff --git a/applications/functions/escrow-xput/include/utils.h b/applications/functions/escrow-xput/include/utils.h new file mode 100644 index 0000000..c410975 --- /dev/null +++ b/applications/functions/escrow-xput/include/utils.h @@ -0,0 +1,5 @@ +#pragma once + +#include + +std::string base64decode(const std::string &data); diff --git a/applications/functions/escrow-xput/src/logger.cpp b/applications/functions/escrow-xput/src/logger.cpp new file mode 100644 index 0000000..b6df46f --- /dev/null +++ b/applications/functions/escrow-xput/src/logger.cpp @@ -0,0 +1,22 @@ +#include "logger.h" + +#include +#include +#include + +void Logger::Log(const char *log_tag, LogLevel level, const char *function, + const int line, const char *fmt, ...) { + va_list args; + va_start(args, fmt); + size_t len = std::vsnprintf(NULL, 0, fmt, args); + va_end(args); + + std::vector str(len + 1); + + va_start(args, fmt); + std::vsnprintf(&str[0], len + 1, fmt, args); + va_end(args); + + // Uncomment for debug logs + // std::cout << std::string(str.begin(), str.end()) << std::endl; +} diff --git a/applications/functions/escrow-xput/src/main.cpp b/applications/functions/escrow-xput/src/main.cpp new file mode 100644 index 0000000..c3a590a --- /dev/null +++ b/applications/functions/escrow-xput/src/main.cpp @@ -0,0 +1,604 @@ +#include "accless/abe4/abe4.h" +#include "accless/attestation/attestation.h" +#include "accless/jwt/jwt.h" + +#include + +/* +#include "AttestationClient.h" +#include "AttestationClientImpl.h" +#include "AttestationParameters.h" +#include "HclReportParser.h" +#include "TpmCertOperations.h" + +#include "logger.h" +#include "tless_abe.h" +#include "utils.h" + +using json = nlohmann::json; + +using namespace attest; + +std::vector split(const std::string &str, char delim) { + std::vector result; + std::stringstream ss(str); + std::string token; + + while (std::getline(ss, token, delim)) { + result.push_back(token); + } + + return result; +} + +// Get the URL of our own attestation service (**not** MAA) +std::string getAttestationServiceUrl() { + const char *val = std::getenv("AS_URL"); + return val ? std::string(val) : "https://127.0.0.1:8443"; +} + +void tpmRenewAkCert() { + TpmCertOperations tpmCertOps; + bool renewalRequired = false; + auto result = tpmCertOps.IsAkCertRenewalRequired(renewalRequired); + if (result.code_ != AttestationResult::ErrorCode::SUCCESS) { + std::cerr << "accless: error checking AkCert renewal state" + << std::endl; + + if (result.tpm_error_code_ != 0) { + std::cerr << "accless: internal TPM error occured: " + << result.description_ << std::endl; + throw std::runtime_error("internal TPM error"); + } else if (result.code_ == attest::AttestationResult::ErrorCode:: + ERROR_AK_CERT_PROVISIONING_FAILED) { + std::cerr << "accless: attestation key cert provisioning delayed" + << std::endl; + throw std::runtime_error("internal TPM error"); + } + } + + if (renewalRequired) { + auto replaceResult = tpmCertOps.RenewAndReplaceAkCert(); + if (replaceResult.code_ != AttestationResult::ErrorCode::SUCCESS) { + std::cerr << "accless: failed to renew AkCert: " + << result.description_ << std::endl; + throw std::runtime_error("accless: internal TPM error"); + } + } +} + +AttestationResult parseClientPayload( + const unsigned char *clientPayload, + std::unordered_map &clientPayloadMap) { + AttestationResult result(AttestationResult::ErrorCode::SUCCESS); + assert(clientPayload != nullptr); + + Json::Value root; + Json::Reader reader; + std::string clientPayloadStr( + const_cast(reinterpret_cast(clientPayload))); + bool success = reader.parse(clientPayloadStr, root); + if (!success) { + std::cout << "accless: error parsing the client payload JSON" + << std::endl; + result.code_ = + AttestationResult::ErrorCode::ERROR_INVALID_INPUT_PARAMETER; + result.description_ = std::string("Invalid client payload Json"); + return result; + } + + for (Json::Value::iterator it = root.begin(); it != root.end(); ++it) { + clientPayloadMap[it.key().asString()] = it->asString(); + } + + return result; +} + +AttestationParameters +getAzureAttestationParameters(AttestationClient *attestationClient) { + std::string attestationUrl = "https://accless.eus.attest.azure.net"; + + // Client parameters + attest::ClientParameters clientParams = {}; + clientParams.attestation_endpoint_url = + (unsigned char *)attestationUrl.c_str(); + // TODO: can we add a public key here? + std::string nonce = "foo"; + std::string clientPayload = "{\"nonce\":\"" + nonce + "\"}"; + clientParams.client_payload = (unsigned char *)clientPayload.c_str(); + clientParams.version = CLIENT_PARAMS_VERSION; + + AttestationParameters params = {}; + std::unordered_map clientPayloadMap; + if (clientParams.client_payload != nullptr) { + auto result = + parseClientPayload(clientParams.client_payload, clientPayloadMap); + if (result.code_ != AttestationResult::ErrorCode::SUCCESS) { + std::cout << "accless: error parsing client payload" << std::endl; + throw std::runtime_error("error parsing client payload"); + } + } + + auto result = ((AttestationClientImpl *)attestationClient) + ->getAttestationParameters(clientPayloadMap, params); + if (result.code_ != AttestationResult::ErrorCode::SUCCESS) { + std::cout << "accless: failed to get attestation parameters" + << std::endl; + throw std::runtime_error("failed to get attestation parameters"); + } + + return params; +} + +std::string maaGetJwtFromParams(AttestationClient *attestationClient, + const AttestationParameters ¶ms, + const std::string &attestationUri) { + bool is_cvm = false; + bool attestation_success = true; + std::string jwt_str; + + unsigned char *jwt = nullptr; + auto attResult = ((AttestationClientImpl *)attestationClient) + ->Attest(params, attestationUri, &jwt); + if (attResult.code_ != attest::AttestationResult::ErrorCode::SUCCESS) { + std::cerr + << "accless: error getting attestation from attestation client" + << std::endl; + Uninitialize(); + throw std::runtime_error( + "failed to get attestation from attestation client"); + } + + std::string jwtStr = reinterpret_cast(jwt); + attestationClient->Free(jwt); + + return jwtStr; +} + +size_t curlWriteCallback(char *ptr, size_t size, size_t nmemb, void *userdata) { + size_t totalSize = size * nmemb; + auto *response = static_cast(userdata); + response->append(ptr, totalSize); + return totalSize; +} + +std::string asGetJwtFromReport(const std::string &asUrl, + const std::vector &snpReport) { + std::string jwt; + + CURL *curl = curl_easy_init(); + if (!curl) { + std::cerr << "accless: failed to initialize CURL" << std::endl; + throw std::runtime_error("curl error"); + } + + curl_easy_setopt(curl, CURLOPT_URL, asUrl.c_str()); + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1L); + curl_easy_setopt( + curl, CURLOPT_CAINFO, + "/home/tless/git/faasm/tless/attestation-service/certs/cert.pem"); + curl_easy_setopt(curl, CURLOPT_POST, 1L); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, snpReport.data()); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, snpReport.size()); + + struct curl_slist *headers = nullptr; + headers = + curl_slist_append(headers, "Content-Type: application/octet-stream"); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + + // Set write function and data + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curlWriteCallback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &jwt); + + // Perform the request + CURLcode res = curl_easy_perform(curl); + if (res != CURLE_OK) { + std::cerr << "accless: CURL error: " << curl_easy_strerror(res) + << std::endl; + curl_easy_cleanup(curl); + curl_slist_free_all(headers); + throw std::runtime_error("curl error"); + } + + curl_easy_cleanup(curl); + curl_slist_free_all(headers); + + return jwt; +} + +void validateJwtClaims(const std::string &jwtStr, bool verbose = false) { + // Prase attestation token to extract isolation tee details + auto tokens = split(jwtStr, '.'); + if (tokens.size() < 3) { + std::cerr << "accless: error validating jwt: not enough tokens" + << std::endl; + throw std::runtime_error("accless: error validating jwt"); + } + + json attestationClaims = json::parse(base64decode(tokens[1])); + std::string attestationType; + std::string complianceStatus; + try { + attestationType = + attestationClaims["x-ms-isolation-tee"]["x-ms-attestation-type"] + .get(); + complianceStatus = + attestationClaims["x-ms-isolation-tee"]["x-ms-compliance-status"] + .get(); + } catch (...) { + std::cerr << "accless: jwt does not have the expected claims" + << std::endl; + throw std::runtime_error("accless: error validating jwt"); + } + + if (!((attestationType == "sevsnpvm") && + (complianceStatus == "azure-compliant-cvm"))) { + std::cerr << "accless: jwt validation does not pass" << std::endl; + } + + if (verbose) { + std::cout << "accless: jwt validation passed" << std::endl; + } +} + +std::vector getSnpReportFromTPM() { + // First, get HCL report + Tpm tpm; + Buffer hclReport = tpm.GetHCLReport(); + + Buffer snpReport; + Buffer runtimeData; + HclReportParser reportParser; + + auto result = reportParser.ExtractSnpReportAndRuntimeDataFromHclReport( + hclReport, snpReport, runtimeData); + if (result.code_ != AttestationResult::ErrorCode::SUCCESS) { + std::cerr << "accless: error parsing snp report from HCL report" + << std::endl; + throw std::runtime_error("error parsing HCL report"); + } + + return snpReport; +} + +void decrypt(const std::string &jwtStr, tless::abe::CpAbeContextWrapper &ctx, + std::vector &cipherText, bool compare = false) { + // TODO: in theory, the attributes should be extracted frm the JWT + std::vector attributes = {"foo", "bar"}; + + auto actualPlainText = ctx.cpAbeDecrypt(attributes, cipherText); + if (actualPlainText.empty()) { + std::cerr << "accless: error decrypting cipher-text" << std::endl; + throw std::runtime_error("error decrypting secret"); + } + + if (compare) { + // Compare + std::string plainText = + "dance like no one's watching, encrypt like everyone is!"; + std::string actualPlainTextStr; + actualPlainTextStr.assign( + reinterpret_cast(actualPlainText.data()), + actualPlainText.size()); + if (actualPlainTextStr == plainText) { + std::cout << "accless: key-release succeeded" << std::endl; + } + std::cout << "accless: actual plain-text: " << actualPlainTextStr + << std::endl; + } +} + +// TODO: do another benchmark where we query our attestation service instead, +// and compare it with the MAA +std::chrono::duration runRequests(int numRequests, int maxParallelism, + bool maa = false) { + // ---------------------- Set Up CP-ABE ----------------------------------- + + // Initialize CP-ABE ctx and create a sample secret + auto &ctx = tless::abe::CpAbeContextWrapper::get( + tless::abe::ContextFetchMode::Create); + std::string plainText = + "dance like no one's watching, encrypt like everyone is!"; + std::string policy = "\"foo\" and \"bar\""; + auto cipherText = ctx.cpAbeEncrypt(policy, plainText); + + // Renew vTPM certificates if needed + tpmRenewAkCert(); + + // ----------------------- Benchmark ------------------------------------- + + std::counting_semaphore semaphore(maxParallelism); + std::vector threads; + auto start = std::chrono::steady_clock::now(); + + if (maa) { + // FIXME: the MAA benchmark has some spurious race conditions + + std::string attestationUri = "https://accless.eus.attest.azure.net"; + + // Initialize Azure Attestation client + AttestationClient *attestationClient = nullptr; + Logger *logHandle = new Logger(); + if (!Initialize(logHandle, &attestationClient)) { + std::cerr << "accless: failed to create attestation client object" + << std::endl; + Uninitialize(); + throw std::runtime_error( + "failed to create attestation client object"); + } + + // Fetching the vTPM measurements is not thread-safe, but would happen + // in each client anyway, so we execute it only once, but still measure + // the time it takes + auto attParams = getAzureAttestationParameters(attestationClient); + + // In the loop, to measure scalability, we only send the HW report for + // validation with the attestation service (be it Azure or our own att. + // service) + for (int i = 1; i < numRequests; ++i) { + semaphore.acquire(); + threads.emplace_back([&semaphore, attestationClient, &attParams, + attestationUri]() { + // Validate some of the claims in the JWT + auto jwtStr = maaGetJwtFromParams(attestationClient, attParams, + attestationUri); + + // TODO: validate JWT signature + + // TODO: somehow get the public key from the JWT + validateJwtClaims(jwtStr); + + // Release semaphore + semaphore.release(); + }); + } + + // Do it once from the main thread to store the return value for + // decryption + auto jwtStr = + maaGetJwtFromParams(attestationClient, attParams, attestationUri); + + for (auto &t : threads) { + if (t.joinable()) { + t.join(); + } + } + + // Similarly, the decrypt stage is compute-bound, so by running many + // instances in parallel we are saturating the local CPU. This step is + // fully distributed, so no issue with running it just once + decrypt(jwtStr, ctx, cipherText); + + Uninitialize(); + } else { + std::string asUrl = getAttestationServiceUrl(); + + // Fetching the vTPM measurements is not thread-safe, but would happen + // in each client anyway, so we execute it only once, but still measure + // the time it takes + auto snpReport = getSnpReportFromTPM(); + + // In the loop, to measure scalability, we only send the HW report for + // validation with the attestation service (be it Azure or our own att. + // service) + for (int i = 1; i < numRequests; ++i) { + semaphore.acquire(); + threads.emplace_back([&semaphore, &asUrl, &snpReport]() { + // Get a JWT from the attestation service if report valid + auto jwtStr = + asGetJwtFromReport(asUrl + "/verify-snp-report", snpReport); + + // TODO: somehow get the public key from the JWT + // TODO: validate some claims in the JWT + + // Release semaphore + semaphore.release(); + }); + } + + // Do it once from the main thread to store the return value for + // decryption + auto jwtStr = + asGetJwtFromReport(asUrl + "/verify-snp-report", snpReport); + + for (auto &t : threads) { + if (t.joinable()) { + t.join(); + } + } + + // Similarly, the decrypt stage is compute-bound, so by running many + // instances in parallel we are saturating the local CPU. This step is + // fully distributed, so no issue with running it just once + decrypt(jwtStr, ctx, cipherText); + } + + auto end = std::chrono::steady_clock::now(); + std::chrono::duration elapsedSecs = end - start; + std::cout << "Elapsed time (" << numRequests << "): " << elapsedSecs.count() + << " seconds\n"; + + return elapsedSecs; +} + +void doBenchmark(bool maa = false) { + // Write elapsed time to CSV + std::string fileName = maa ? "accless-maa.csv" : "accless.csv"; + std::ofstream csvFile(fileName, std::ios::out); + csvFile << "NumRequests,TimeElapsed\n"; + + // WARNING: this is copied from invrs/src/tasks/ubench.rs and must be + // kept in sync! + std::vector numRequests = {1, 10, 50, 100, 200, 400, 600, 800, 1000}; + int numRepeats = maa ? 1 : 3; + int maxParallelism = 100; + try { + for (const auto &i : numRequests) { + for (int j = 0; j < numRepeats; j++) { + auto elapsedTimeSecs = runRequests(i, maxParallelism, maa); + csvFile << i << "," << elapsedTimeSecs.count() << '\n'; + } + } + } catch (...) { + std::cout << "accless: error running benchmark" << std::endl; + } + + csvFile.close(); +} + +void runOnce(bool maa = false) { + // Renew TPM certificates if needed + tpmRenewAkCert(); + + // Initialize CP-ABE ctx + auto &ctx = tless::abe::CpAbeContextWrapper::get( + tless::abe::ContextFetchMode::Create); + std::string plainText = + "dance like no one's watching, encrypt like everyone is!"; + std::string policy = "\"foo\" and \"bar\""; + auto cipherText = ctx.cpAbeEncrypt(policy, plainText); + + std::string jwtStr; + if (maa) { + // TODO: attest MAA + std::string attestationUri = "https://accless.eus.attest.azure.net"; + + // Initialize Azure Attestation client + AttestationClient *attestationClient = nullptr; + Logger *logHandle = new Logger(); + if (!Initialize(logHandle, &attestationClient)) { + std::cerr << "accless: failed to create attestation client object" + << std::endl; + Uninitialize(); + throw std::runtime_error( + "failed to create attestation client object"); + } + + auto attParams = getAzureAttestationParameters(attestationClient); + jwtStr = + maaGetJwtFromParams(attestationClient, attParams, attestationUri); + validateJwtClaims(jwtStr); + + Uninitialize(); + } else { + std::string asUrl = getAttestationServiceUrl(); + + // TODO: attest AS + + auto snpReport = getSnpReportFromTPM(); + jwtStr = asGetJwtFromReport(asUrl + "/verify-snp-report", snpReport); + std::cout << "out: " << jwtStr << std::endl; + } + + // TODO: jwtStr is now a JWE, so we must decrypt it + + decrypt(jwtStr, ctx, cipherText); +} +*/ + +/** + * @brief Performs a single secret-key-release operation using Accless. + * + * This function is the main body of Accless secret-key-release operation. It + * relies on an instance of the attestation-service running, and on being + * deployed in a genuine SNP cVM. Either in a para-virtualized environment on + * Azure, or on bare-metal. + */ +int doAcclessSkr() { + // Get the ID and MPK we need to encrypt ciphertexts with attributes from + // this attestation service instance. + auto [id, partialMpk] = accless::attestation::getAttestationServiceState(); + std::cout << "escrow-xput: got attesation service's state" << std::endl; + std::string mpk = accless::abe4::packFullKey({id}, {partialMpk}); + std::cout << "escrow-xput: packed partial MPK into full MPK" << std::endl; + + std::string gid = "baz"; + std::string wfId = "foo"; + std::string nodeId = "bar"; + + // Pick the simplest policy that only relies on the attributes `wf` and + // `node` which are provided by the attestation-service after a succesful + // remote attestation. + std::string policy = id + ".wf:" + wfId + " & " + id + ".node:" + nodeId; + + // Generate a test ciphertext that only us, after a succesful attestation, + // should be able to decrypt. + std::cout << "escrow-xput: encrypting cp-abe with policy: " << policy + << std::endl; + auto [gt, ct] = accless::abe4::encrypt(mpk, policy); + if (gt.empty() || ct.empty()) { + std::cerr << "escrow-xput: error running cp-abe encryption" + << std::endl; + return 1; + } + std::cout << "escrow-xput: ran CP-ABE encryption" << std::endl; + + // TODO: Should start measuring now! + std::cout << "escrow-xput: running remote attestation..." << std::endl; + try { + const std::string jwt = + accless::attestation::snp::getAttestationJwt(gid, wfId, nodeId); + if (jwt.empty()) { + std::cerr << "escrow-xput: empty JWT returned" << std::endl; + return 1; + } + + if (!accless::jwt::verify(jwt)) { + std::cerr << "escrow-xput: JWT signature verification failed" + << std::endl; + return 1; + } + + // Get the partial USK from the JWT, and wrap it in a full key for + // CP-ABE decryption. + std::string partialUskB64 = + accless::jwt::getProperty(jwt, "partial_usk_b64"); + if (partialUskB64.empty()) { + std::cerr + << "att-client-snp: JWT is missing 'partial_usk_b64' field" + << std::endl; + return 1; + } + std::string uskB64 = accless::abe4::packFullKey({id}, {partialUskB64}); + + // Run decryption. + std::optional decrypted_gt = + accless::abe4::decrypt(uskB64, gid, policy, ct); + if (!decrypted_gt.has_value()) { + std::cerr << "att-client-snp: CP-ABE decryption failed" + << std::endl; + return 1; + } else if (decrypted_gt.value() != gt) { + std::cerr << "att-client-snp: CP-ABE decrypted ciphertexts do not" + << " match!" << std::endl; + std::cerr << "att-client-snp: Original GT: " << gt << std::endl; + std::cerr << "att-client-snp: Decrypted GT: " + << decrypted_gt.value() << std::endl; + return 1; + } + + // End of experiment. + } catch (const std::exception &ex) { + std::cerr << "escrow-xput: error: " << ex.what() << std::endl; + } catch (...) { + std::cerr << "escrow-xput: unexpected error" << std::endl; + } + + std::cout << "escrow-xput: experiment succesful" << std::endl; + return 0; +} + +int main(int argc, char **argv) { + bool maa = ((argc == 2) && (std::string(argv[1]) == "--maa")); + bool once = ((argc == 2) && (std::string(argv[1]) == "--once")); + + doAcclessSkr(); + + /* + if (once) { + runOnce(); + } else { + doBenchmark(maa); + } + */ +} diff --git a/applications/functions/escrow-xput/src/utils.cpp b/applications/functions/escrow-xput/src/utils.cpp new file mode 100644 index 0000000..eec2fff --- /dev/null +++ b/applications/functions/escrow-xput/src/utils.cpp @@ -0,0 +1,14 @@ +#include "utils.h" + +#include +#include +#include + +std::string base64decode(const std::string &data) { + using namespace boost::archive::iterators; + using It = + transform_width, 8, 6>; + return boost::algorithm::trim_right_copy_if( + std::string(It(std::begin(data)), It(std::end(data))), + [](char c) { return c == '\0'; }); +} From f4dc3933b19646752d8f1af8eabe9bc00e3c1021 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Mon, 24 Nov 2025 15:41:45 +0000 Subject: [PATCH 12/52] [attestation-service] E: Verify Bare-Metal SNP Reports Closes #25 --- Cargo.lock | 210 +++++++++--------------------- Cargo.toml | 2 +- GEMINI.md | 13 +- attestation-service/Cargo.toml | 3 +- attestation-service/src/amd.rs | 122 ++++++++++++++++++ attestation-service/src/lib.rs | 24 ++-- attestation-service/src/main.rs | 4 +- attestation-service/src/sgx.rs | 2 +- attestation-service/src/snp.rs | 212 +++++++++++++++++++++++++++++-- attestation-service/src/state.rs | 52 ++++++-- 10 files changed, 455 insertions(+), 189 deletions(-) create mode 100644 attestation-service/src/amd.rs diff --git a/Cargo.lock b/Cargo.lock index d2a03b2..d407361 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -475,11 +475,11 @@ dependencies = [ "serde", "serde_json", "serial_test", + "sev", "snpguest", "tempfile", "tokio", "tokio-rustls", - "ureq", ] [[package]] @@ -537,7 +537,7 @@ dependencies = [ "serde_json", "serde_path_to_error", "serde_urlencoded", - "sync_wrapper 1.0.2", + "sync_wrapper", "tokio", "tower", "tower-layer", @@ -560,7 +560,7 @@ dependencies = [ "mime", "pin-project-lite", "rustversion", - "sync_wrapper 1.0.2", + "sync_wrapper", "tower-layer", "tower-service", "tracing", @@ -572,6 +572,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "base64" version = "0.21.7" @@ -625,6 +631,26 @@ version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c821a6e124197eb56d907ccc2188eab1038fb919c914f47976e64dd8dbc855d1" +[[package]] +name = "bitfield" +version = "0.19.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21ba6517c6b0f2bf08be60e187ab64b038438f22dd755614d8fe4d4098c46419" +dependencies = [ + "bitfield-macros", +] + +[[package]] +name = "bitfield-macros" +version = "0.19.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f48d6ace212fdf1b45fd6b566bb40808415344642b76c3224c07c8df9da81e97" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.108", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -876,12 +902,6 @@ dependencies = [ "cc", ] -[[package]] -name = "codicon" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12170080f3533d6f09a19f81596f836854d0fa4867dc32c8172b8474b4e9de61" - [[package]] name = "color_quant" version = "1.1.0" @@ -1277,34 +1297,13 @@ dependencies = [ "subtle", ] -[[package]] -name = "dirs" -version = "5.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" -dependencies = [ - "dirs-sys 0.4.1", -] - [[package]] name = "dirs" version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" dependencies = [ - "dirs-sys 0.5.0", -] - -[[package]] -name = "dirs-sys" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" -dependencies = [ - "libc", - "option-ext", - "redox_users 0.4.6", - "windows-sys 0.48.0", + "dirs-sys", ] [[package]] @@ -1315,7 +1314,7 @@ checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" dependencies = [ "libc", "option-ext", - "redox_users 0.5.2", + "redox_users", "windows-sys 0.61.2", ] @@ -1546,7 +1545,7 @@ dependencies = [ "core-foundation", "core-graphics", "core-text", - "dirs 6.0.0", + "dirs", "dwrote", "float-ord", "freetype-sys", @@ -2109,7 +2108,7 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", - "webpki-roots 1.0.3", + "webpki-roots", ] [[package]] @@ -2160,7 +2159,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "socket2 0.6.1", - "system-configuration 0.6.1", + "system-configuration", "tokio", "tower-service", "tracing", @@ -2370,7 +2369,7 @@ dependencies = [ "socket2 0.5.10", "widestring", "windows-sys 0.48.0", - "winreg", + "winreg 0.50.0", ] [[package]] @@ -3446,17 +3445,6 @@ dependencies = [ "bitflags 2.10.0", ] -[[package]] -name = "redox_users" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" -dependencies = [ - "getrandom 0.2.16", - "libredox", - "thiserror 1.0.69", -] - [[package]] name = "redox_users" version = "0.5.2" @@ -3499,11 +3487,11 @@ checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" [[package]] name = "reqwest" -version = "0.11.27" +version = "0.11.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" +checksum = "46a1f7aa4f35e5e8b4160449f51afc758f0ce6454315a9fa7d0d113e958c41eb" dependencies = [ - "base64 0.21.7", + "base64 0.13.1", "bytes", "encoding_rs", "futures-core", @@ -3515,26 +3503,22 @@ dependencies = [ "hyper-tls 0.5.0", "ipnet", "js-sys", + "lazy_static", "log", "mime", "native-tls", - "once_cell", "percent-encoding", "pin-project-lite", - "rustls-pemfile", "serde", "serde_json", "serde_urlencoded", - "sync_wrapper 0.1.2", - "system-configuration 0.5.1", "tokio", "tokio-native-tls", - "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "winreg", + "winreg 0.10.1", ] [[package]] @@ -3571,7 +3555,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "sync_wrapper 1.0.2", + "sync_wrapper", "tokio", "tokio-native-tls", "tokio-rustls", @@ -3584,7 +3568,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots 1.0.3", + "webpki-roots", ] [[package]] @@ -3853,15 +3837,6 @@ dependencies = [ "serde_derive", ] -[[package]] -name = "serde-big-array" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11fc7cc2c76d73e0f27ee52abbd64eec84d46f370c88371120433196934e4b7f" -dependencies = [ - "serde", -] - [[package]] name = "serde-human-bytes" version = "0.1.1" @@ -3872,16 +3847,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_bytes" -version = "0.11.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" -dependencies = [ - "serde", - "serde_core", -] - [[package]] name = "serde_core" version = "1.0.228" @@ -3979,26 +3944,21 @@ dependencies = [ [[package]] name = "sev" -version = "5.0.0" +version = "7.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b06afe5192a43814047ea0072f4935f830a1de3c8cb43b56c90ae6918468b94d" +checksum = "c2ff74d7e7d1cc172f3a45adec74fbeee928d71df095b85aaaf66eb84e1e31e6" dependencies = [ "base64 0.22.1", - "bincode", - "bitfield", - "bitflags 1.3.2", + "bitfield 0.19.4", + "bitflags 2.10.0", "byteorder", - "codicon", - "dirs 5.0.1", + "dirs", "hex", "iocuddle", "lazy_static", "libc", "openssl", "rdrand", - "serde", - "serde-big-array", - "serde_bytes", "static_assertions", "uuid", ] @@ -4037,7 +3997,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b1fdf65dd6331831494dd616b30351c38e96e45921a27745cf98490458b90bb" dependencies = [ - "dirs 6.0.0", + "dirs", ] [[package]] @@ -4118,14 +4078,13 @@ dependencies = [ [[package]] name = "snpguest" -version = "0.8.3" -source = "git+https://github.com/faasm/snpguest.git#d3697058d4981db9c8dca1f6321cab3ca5cb029d" +version = "0.10.0" dependencies = [ "anyhow", "asn1-rs", "base64 0.22.1", "bincode", - "bitfield", + "bitfield 0.15.0", "clap", "clap-num", "colorful", @@ -4135,7 +4094,7 @@ dependencies = [ "nix", "openssl", "rand 0.8.5", - "reqwest 0.11.27", + "reqwest 0.11.10", "serde", "sev", "x509-parser", @@ -4223,12 +4182,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "sync_wrapper" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" - [[package]] name = "sync_wrapper" version = "1.0.2" @@ -4249,17 +4202,6 @@ dependencies = [ "syn 2.0.108", ] -[[package]] -name = "system-configuration" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" -dependencies = [ - "bitflags 1.3.2", - "core-foundation", - "system-configuration-sys 0.5.0", -] - [[package]] name = "system-configuration" version = "0.6.1" @@ -4268,17 +4210,7 @@ checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" dependencies = [ "bitflags 2.10.0", "core-foundation", - "system-configuration-sys 0.6.0", -] - -[[package]] -name = "system-configuration-sys" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" -dependencies = [ - "core-foundation-sys", - "libc", + "system-configuration-sys", ] [[package]] @@ -4559,7 +4491,7 @@ dependencies = [ "futures-core", "futures-util", "pin-project-lite", - "sync_wrapper 1.0.2", + "sync_wrapper", "tokio", "tower-layer", "tower-service", @@ -4711,24 +4643,6 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" -[[package]] -name = "ureq" -version = "2.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" -dependencies = [ - "base64 0.22.1", - "flate2", - "log", - "once_cell", - "rustls", - "rustls-pki-types", - "serde", - "serde_json", - "url", - "webpki-roots 0.26.11", -] - [[package]] name = "url" version = "2.5.7" @@ -4957,15 +4871,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "webpki-roots" -version = "0.26.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" -dependencies = [ - "webpki-roots 1.0.3", -] - [[package]] name = "webpki-roots" version = "1.0.3" @@ -5352,6 +5257,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "winreg" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +dependencies = [ + "winapi", +] + [[package]] name = "winreg" version = "0.50.0" diff --git a/Cargo.toml b/Cargo.toml index 649a1d3..ba8956a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -65,6 +65,7 @@ serde = { version = "^1.0", features = ["derive"] } serde_json = "^1.0" serde_with = "3.8.1" serde_yaml = "0.9" +sev = "7.1.0" shellexpand = "^3.1" sha2 = "0.10" shell-words = "^1.1.0" @@ -73,7 +74,6 @@ subtle = "2.6.1" tempfile = "3.23.0" tokio = { version = "1" } tokio-rustls = "0.26.2" -ureq = { version = "2" } uuid = { version = "^1.3" } walkdir = "2" warp = "^0.3" diff --git a/GEMINI.md b/GEMINI.md index 0e0bc1b..e989544 100644 --- a/GEMINI.md +++ b/GEMINI.md @@ -84,10 +84,10 @@ optional `--help` flag: `./scrips/accli_wrapper.sh --help`. error handling through the `anyhow` crate. - For each new method, make sure to add extensive documentation in the following format: ```rust +/// /// -/// # Description -/// -/// +/// +/// /// /// # Arguments /// @@ -102,6 +102,13 @@ optional `--help` flag: `./scrips/accli_wrapper.sh --help`. /// /// ASK --(signs)--> +/// VCEK --(signs)--> Report +pub type SnpCa = Chain; + +/// SNP-enabled processor type. +pub type SnpProcType = ProcType; + +/// SNP attestation report +pub type SnpReport = AttestationReport; + +/// Vendor Chip Endorsement Key +pub type SnpVcek = Certificate; + +/// # Description +/// +/// We cache VCEK certificates to validate SNP reports by the processor type, +/// and the reported TCB. Note that even though the TCB version is +/// self-reported, it is included in the report and signed by the PSP. +pub type SnpVcekCacheKey = (SnpProcType, TcbVersion); + +const AMD_KDS_SITE: &str = "https://kdsintf.amd.com"; + +fn proc_type_to_kds_url(proc_type: &SnpProcType) -> &str { + match proc_type { + SnpProcType::Genoa | SnpProcType::Siena | SnpProcType::Bergamo => "Genoa", + SnpProcType::Milan => "Milan", + SnpProcType::Turin => "Turin", + } +} + +/// Fetches AMD's ceritifcate authorities from AMD's Key Distribution Service. +pub async fn fetch_ca_from_kds(proc_type: &SnpProcType) -> Result { + const AMD_KDS_CERT_CHAIN: &str = "cert_chain"; + + let proc_str = proc_type_to_kds_url(proc_type); + let url: String = format!("{AMD_KDS_SITE}/vcek/v1/{proc_str}/{AMD_KDS_CERT_CHAIN}"); + trace!("fetch_ca_from_kds(): fetching AMD's CA (url={url})"); + let client = reqwest::Client::new(); + let response = client.get(url).send().await?; + let response = response.error_for_status()?; + + let body = response.bytes().await?; + let ca_chain = Chain::from_pem_bytes(&body)?; + + // Before returning it, verify the signatures. + ca_chain.verify()?; + trace!("fetch_ca_from_kds(): verified CA's signature"); + + Ok(ca_chain) +} + +/// Fetches a processor's Vendor-Chip Endorsement Key (VCEK) from AMD's KDS. +pub async fn fetch_vcek_from_kds( + proc_type: &SnpProcType, + att_report: &SnpReport, +) -> Result { + const KDS_VCEK: &str = "/vcek/v1"; + + // The URL generation part in this function is adapted from the snpguest crate. + let hw_id: String = if att_report.chip_id != [0; 64] { + match proc_type { + ProcType::Turin => { + let shorter_bytes: &[u8] = &att_report.chip_id[0..8]; + hex::encode(shorter_bytes) + } + _ => hex::encode(att_report.chip_id), + } + } else { + let reason = "fetch_vcek_from_kds(): hardware ID is 0s on attestation report"; + error!("{reason}"); + anyhow::bail!(reason); + }; + let url: String = match proc_type { + ProcType::Turin => { + let fmc = if let Some(fmc) = att_report.reported_tcb.fmc { + fmc + } else { + return Err(anyhow::anyhow!("A Turin processor must have a fmc value")); + }; + format!( + "{AMD_KDS_SITE}{KDS_VCEK}/{}/\ + {hw_id}?fmcSPL={:02}&blSPL={:02}&teeSPL={:02}&snpSPL={:02}&ucodeSPL={:02}", + proc_type_to_kds_url(proc_type), + fmc, + att_report.reported_tcb.bootloader, + att_report.reported_tcb.tee, + att_report.reported_tcb.snp, + att_report.reported_tcb.microcode + ) + } + _ => { + format!( + "{AMD_KDS_SITE}{KDS_VCEK}/{}/\ + {hw_id}?blSPL={:02}&teeSPL={:02}&snpSPL={:02}&ucodeSPL={:02}", + proc_type_to_kds_url(proc_type), + att_report.reported_tcb.bootloader, + att_report.reported_tcb.tee, + att_report.reported_tcb.snp, + att_report.reported_tcb.microcode + ) + } + }; + + trace!("fetch_vcek_from_kds(): fetching node's VCEK (url={url})"); + let client = reqwest::Client::new(); + let response = client.get(url).send().await?; + let response = response.error_for_status()?; + + let body = response.bytes().await?; + let vcek = Certificate::from_bytes(&body)?; + + Ok(vcek) +} diff --git a/attestation-service/src/lib.rs b/attestation-service/src/lib.rs index 6c62e7f..cac74d1 100644 --- a/attestation-service/src/lib.rs +++ b/attestation-service/src/lib.rs @@ -1,20 +1,20 @@ -use env_logger::Env; +use env_logger::{Builder, Env}; +use log::LevelFilter; use std::sync::Once; static INIT: Once = Once::new(); -pub fn init_logging(is_test: bool) { +pub fn init_logging() { INIT.call_once(|| { - let default_filter = if is_test { - // In tests, be more chatty by default. - "info,attestation_service=info,accli=info" - } else { - // In normal runs, keep everything else at error. - "error,attestation_service=info,accli=info" - }; + let env = Env::default().filter_or("RUST_LOG", "info"); + let mut builder = Builder::from_env(env); - let _ = env_logger::Builder::from_env(Env::default().default_filter_or(default_filter)) - .is_test(is_test) - .try_init(); + // Disable noisy modules. + let noisy_modules: Vec<&str> = vec!["hickory_proto", "hyper_util", "hyper", "reqwest"]; + for module in &noisy_modules { + builder.filter_module(module, LevelFilter::Off); + } + + builder.init(); }); } diff --git a/attestation-service/src/main.rs b/attestation-service/src/main.rs index ebe4d97..ab83cfd 100644 --- a/attestation-service/src/main.rs +++ b/attestation-service/src/main.rs @@ -14,6 +14,8 @@ use rustls::crypto::CryptoProvider; use std::{net::SocketAddr, path::PathBuf, sync::Arc}; use tokio::net::TcpListener; +#[cfg(feature = "snp")] +mod amd; #[cfg(feature = "azure-cvm")] mod azure_cvm; mod ecdhe; @@ -59,7 +61,7 @@ async fn health() -> impl IntoResponse { async fn main() -> Result<()> { // Initialise logging and parse CLI arguments. let cli = Cli::parse(); - attestation_service::init_logging(false); + attestation_service::init_logging(); // Initialise crypto provider and TLS config, this also sets up the TLS // certificates if necessary. diff --git a/attestation-service/src/sgx.rs b/attestation-service/src/sgx.rs index 7dd2d14..ba2bb00 100644 --- a/attestation-service/src/sgx.rs +++ b/attestation-service/src/sgx.rs @@ -227,6 +227,7 @@ pub async fn verify_sgx_report( ); } }; + // FIXME(#55): also check the MRENCLAVE measurement against a reference value. let verified_report = match dcap_qvl::verify::verify("e_bytes, &collateral, now) { Ok(tcb) => tcb, Err(e) => { @@ -237,7 +238,6 @@ pub async fn verify_sgx_report( ); } }; - info!("verififed sgx report (status={})", verified_report.status); match verified_report.report { diff --git a/attestation-service/src/snp.rs b/attestation-service/src/snp.rs index a080d42..bbdd0ac 100644 --- a/attestation-service/src/snp.rs +++ b/attestation-service/src/snp.rs @@ -1,15 +1,18 @@ use crate::{ + amd::{SnpCa, SnpProcType, SnpReport, SnpVcek, fetch_ca_from_kds, fetch_vcek_from_kds}, ecdhe, jwt::{self, JwtClaims}, mock::{MockQuote, MockQuoteType}, request::{NodeData, Tee}, state::AttestationServiceState, }; +use anyhow::Result; use axum::{Extension, Json, http::StatusCode, response::IntoResponse}; use base64::{Engine as _, engine::general_purpose}; use log::{debug, error, info}; use serde::Deserialize; use serde_json::json; +use sev::{certs::snp::Verifiable, parser::ByteParser}; use std::sync::Arc; #[derive(Debug, Deserialize)] @@ -26,8 +29,6 @@ struct RuntimeData { _data_type: String, } -/// # Description -/// /// This struct corresponds to the request that SNP-Knative sends to verify an /// SNP attestation report. #[derive(Debug, Deserialize)] @@ -45,6 +46,139 @@ pub struct SnpRequest { runtime_data: RuntimeData, } +/// Extract the report payload from the PSP reposnse. +/// +/// The attestation-service receives from SNP clients the literal response +/// returned by the PSP. The structure of this response is described in Table 25 +/// [1]. We observe that the actual report is padded, so this method extracts +/// the actual attestation report from the PSP response. Note that crates like +/// snpguest manipulate the actual report, and not the PSP response. They would +/// rely on the `sev` crate to do the parsing but, annoyingly, it does not +/// expose a public API for us to parse the PSP response from bytes. +/// +/// [1] https://www.amd.com/content/dam/amd/en/documents/developer/56860.pdf +/// +/// # Arguments +/// +/// - `psp_response`: the raw PSP response from the SNP_GET_REPORT command. +/// +/// # Returns +/// +/// The report byte array within the PSP response. +fn extract_report(data: &[u8]) -> Result> { + const OFFSET_STATUS: usize = 0x00; + const OFFSET_REPORT_SIZE: usize = 0x04; + const OFFSET_REPORT_DATA: usize = 0x20; + + // We need at least 0x20 (32) bytes to reach the report data start. + if data.len() < OFFSET_REPORT_DATA { + let reason = format!( + "PSP response buffer too short to contain header (got={}, minimum={OFFSET_REPORT_DATA})", + data.len() + ); + error!("{reason}"); + anyhow::bail!(reason); + } + + // Check status. + let status_bytes: [u8; 4] = data[OFFSET_STATUS..OFFSET_STATUS + 4].try_into()?; + let status = u32::from_le_bytes(status_bytes); + if status != 0 { + let reason = format!("PSP reported firmware error (error={:#x})", status); + error!("{}", reason); + anyhow::bail!(reason); + } + + // Get the report size. + let size_bytes: [u8; 4] = data[OFFSET_REPORT_SIZE..OFFSET_REPORT_SIZE + 4].try_into()?; + let report_size = u32::from_le_bytes(size_bytes) as usize; + + // Validate that the buffer actually holds the amount of data declared. + let required_len = OFFSET_REPORT_DATA + report_size; + if data.len() < required_len { + let reason = "report is shorter than expected size"; + error!("{}", reason); + anyhow::bail!(reason); + } + + // Extract report. We slice from 0x20 to (0x20 + size) and convert to an owned + // vec. + let report_payload = data[OFFSET_REPORT_DATA..required_len].to_vec(); + Ok(report_payload) +} + +async fn get_snp_ca( + proc_type: &SnpProcType, + state: &Arc, +) -> Result { + debug!("get_snp_ca(): getting CA chain for SNP processor (type={proc_type})"); + + // Fast path: read CA from the cache. + let ca: Option = { + let cache = state.amd_signing_keys.read().await; + cache.get(proc_type).cloned() + }; + + if let Some(ca) = ca { + debug!("get_snp_ca(): cache hit, fetching CA from local cache"); + return Ok(ca); + } + + // This method also verifies the CA signatures. + debug!("get_snp_ca(): cache miss, fetching CA from AMD's KDS"); + let ca = fetch_ca_from_kds(proc_type).await?; + + // Cache CA for future use. + { + let mut cache = state.amd_signing_keys.write().await; + cache.insert(proc_type.clone(), ca.clone()); + } + + Ok(ca) +} + +/// Helper method to fetch the VCEK certificate to validate an SNP quote. We +/// cache the certificates based on the platform and TCB info to avoid +/// round-trips to the AMD servers during verification (in the general case). +async fn get_snp_vcek(report: &SnpReport, state: &Arc) -> Result { + // Fetch the certificate chain from the processor model. + let proc_type = snpguest::fetch::get_processor_model(report)?; + let ca = get_snp_ca(&proc_type, state).await?; + + // Work-out cache key from report. + let tcb_version = report.reported_tcb; + let cache_key = (proc_type.clone(), tcb_version); + debug!( + "get_snp_vcek(): fetching VCEK key for report (proc_type={proc_type}, tcb={tcb_version})" + ); + + // Fast path: read VCEK from the cache. + let vcek: Option = { + let cache = state.snp_vcek_cache.read().await; + cache.get(&cache_key).cloned() + }; + + if let Some(vcek) = vcek { + debug!("get_snp_vcek(): cache hit, fetching VCEK from local cache"); + return Ok(vcek); + } + + // Slow path: fetch collateral from AMD's KDS. + debug!("get_snp_vcek(): cache miss, fetching VCEK from AMD's KDS"); + let vcek = fetch_vcek_from_kds(&proc_type, report).await?; + + // Once we fetch a new VCEK, verify its certificate chain before caching it. + (&ca.ask, &vcek).verify()?; + + // Cache VCEK for future use. + { + let mut cache = state.snp_vcek_cache.write().await; + cache.insert(cache_key, vcek.clone()); + } + + Ok(vcek) +} + pub async fn verify_snp_report( Extension(state): Extension>, Json(payload): Json, @@ -54,7 +188,7 @@ pub async fn verify_snp_report( let quote_bytes = match general_purpose::URL_SAFE.decode(&raw_quote_b64) { Ok(bytes) => bytes, Err(e) => { - error!("invalid base64 string in SNP quote (error={e:?})"); + error!("verify_snp_report(): invalid base64 string in SNP quote (error={e:?})"); return ( StatusCode::BAD_REQUEST, Json(json!({ "error": "invalid base64 in quote" })), @@ -66,17 +200,17 @@ pub async fn verify_snp_report( match MockQuote::from_bytes("e_bytes) { Ok(mock_quote) => { if mock_quote.quote_type != MockQuoteType::Snp { - error!("invalid mock SNP quote (error=wrong quote type)"); + error!("verify_snp_report(): invalid mock SNP quote (error=wrong quote type)"); return ( StatusCode::BAD_REQUEST, Json(json!({ "error": "invalid mock SNP quote" })), ); } - info!("received mock SNP quote, skipping verification"); + info!("verify_snp_report(): received mock SNP quote, skipping verification"); mock_quote.user_data } Err(e) => { - error!("invalid mock SNP quote (error={e:?})"); + error!("verify_snp_report(): invalid mock SNP quote (error={e:?})"); return ( StatusCode::BAD_REQUEST, Json(json!({ "error": "invalid mock SNP quote" })), @@ -84,12 +218,66 @@ pub async fn verify_snp_report( } } } else { - // FIXME(#25): validate SNP reports on bare metal - error!("missing logic to validate SNP reports on bare metal"); - return ( - StatusCode::NOT_IMPLEMENTED, - Json(json!({ "error": "SNP report validation not implemented" })), - ); + // Even though the response from the PSP to SNP_GET_REPORT is padded to 4000 + // bytes [1], the snpguest crate expects the AttestationReport to be the + // exact size in bytes, without padding [2]. We receive from the client + // the raw response from the PSP, so we must remove the padding first. + // The structure of the PSP response can be found in Table 25 [3]. + // + // [1] https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/include/uapi/linux/sev-guest.h + // [2] https://github.com/virtee/sev/blob/c7b6bbb4e9c0fe85199723ab082ccadf39a494f0/src/firmware/linux/guest/types.rs#L169-L183 + // [3] https://www.amd.com/content/dam/amd/en/documents/developer/56860.pdf + let report_body = match extract_report("e_bytes) { + Ok(report_body) => report_body, + Err(e) => { + error!("verify_snp_report(): error extracting report body (error={e:?})"); + return ( + StatusCode::BAD_REQUEST, + Json(json!({ "error": "invalid SNP quote" })), + ); + } + }; + + // Parse the attestation report from bytes. + let report: SnpReport = match SnpReport::from_bytes(&report_body) { + Ok(report) => report, + Err(e) => { + error!("verify_snp_report(): error parsing bytes to SNP report (error={e:?})"); + return ( + StatusCode::BAD_REQUEST, + Json(json!({ "error": "error parsing SNP report" })), + ); + } + }; + + // Fetch the VCEK certificate. + let vcek = match get_snp_vcek(&report, &state).await { + Ok(report) => report, + Err(e) => { + error!("verify_snp_report(): error fetching SNP VCEK (error={e:?})"); + return ( + StatusCode::BAD_REQUEST, + Json(json!({ "error": "error fetching SNP VCEK" })), + ); + } + }; + + // FIXME(#55): also check the SNP measurement against a reference value. + match snpguest::verify::attestation::verify_attestation(&vcek, &report) { + Ok(()) => { + info!("verify_snp_report(): verified SNP report"); + + // Report data to owned vec. + report.report_data.to_vec() + } + Err(e) => { + error!("error verifying SNP report (error={e:?})"); + return ( + StatusCode::BAD_REQUEST, + Json(json!({ "error": "error verifying SNP report" })), + ); + } + } }; // Use the enclave held data (runtime_data) public key, to derive an diff --git a/attestation-service/src/state.rs b/attestation-service/src/state.rs index 5fedbb5..c1daa9a 100644 --- a/attestation-service/src/state.rs +++ b/attestation-service/src/state.rs @@ -1,3 +1,5 @@ +#[cfg(feature = "snp")] +use crate::amd::{SnpCa, SnpProcType, SnpVcek, SnpVcekCacheKey}; #[cfg(feature = "azure-cvm")] use crate::azure_cvm; #[cfg(feature = "sgx")] @@ -7,13 +9,24 @@ use abe4::scheme::types::{PartialMPK, PartialMSK}; use anyhow::Result; #[cfg(feature = "sgx")] use jsonwebtoken::EncodingKey; -use std::{collections::HashMap, path::PathBuf}; +use std::{ + collections::{BTreeMap, HashMap}, + path::PathBuf, +}; use tokio::sync::RwLock; -/// Unique identifier for the demo attestation service. +/// Unique alphanumeric identifier for the demo attestation service. const ATTESTATION_SERVICE_ID: &str = "4CL3SSD3M0"; pub struct AttestationServiceState { + // General attestation service fields. + /// Run the attestation handlers in mock mode, skipping quote verification + /// while still exercising the rest of the request flow. + pub mock_attestation: bool, + /// JWT encoding key derived from the service's public certificate. + pub jwt_encoding_key: EncodingKey, + + // Fields related to attribute-based encryption. /// Unique ID for this attestation service. This is the field that must be /// included in the template graph, and is the field we use to run CP-ABE /// key generation. @@ -24,9 +37,10 @@ pub struct AttestationServiceState { /// Master Pulic Key for the attestation service as one of the authorities /// of the decentralized CP-ABE scheme. pub partial_mpk: PartialMPK, - #[cfg(feature = "azure-cvm")] - pub vcek_pem: Vec, - pub jwt_encoding_key: EncodingKey, + + // Fields related to verifying attestation reports from TEEs. + + // Intel SGX. /// URL to a Provisioning Certificate Caching Service (PCCS) to verify SGX /// quotes. #[cfg(feature = "sgx")] @@ -36,9 +50,21 @@ pub struct AttestationServiceState { /// collaterals. #[cfg(feature = "sgx")] pub sgx_collateral_cache: RwLock>, - /// Run the attestation handlers in mock mode, skipping quote verification - /// while still exercising the rest of the request flow. - pub mock_attestation: bool, + + // Amd SEV-SNP. + #[cfg(feature = "snp")] + /// AMD's root (ARK) and signing (ASK) keys, which make up the ceritificate + /// chain of SNP reports: + pub amd_signing_keys: RwLock>, + /// Cache of VCEK certificates used to validate the signatures of SNP + /// reports. + /// + /// We use a BTreeMap to workaround the lack of Hash traits for the cache + /// key, which is a tuple of types we don't control. + pub snp_vcek_cache: RwLock>, + + #[cfg(feature = "azure-cvm")] + pub vcek_pem: Vec, } impl AttestationServiceState { @@ -57,18 +83,24 @@ impl AttestationServiceState { let (partial_msk, partial_mpk): (PartialMSK, PartialMPK) = abe4::scheme::setup_partial(&mut rng, ATTESTATION_SERVICE_ID); + // Fetch AMD signing keys. + Ok(Self { + mock_attestation, + jwt_encoding_key: jwt::generate_encoding_key(&certs_dir)?, id: ATTESTATION_SERVICE_ID.to_string(), partial_msk, partial_mpk, #[cfg(feature = "azure-cvm")] vceck_pem: azure_cvm::fetch_vcek_pem()?, - jwt_encoding_key: jwt::generate_encoding_key(&certs_dir)?, #[cfg(feature = "sgx")] sgx_pccs_url, #[cfg(feature = "sgx")] sgx_collateral_cache: RwLock::new(HashMap::new()), - mock_attestation, + #[cfg(feature = "snp")] + amd_signing_keys: RwLock::new(BTreeMap::new()), + #[cfg(feature = "snp")] + snp_vcek_cache: RwLock::new(BTreeMap::new()), }) } } From 43df72e6b3cd5ff5be5659d39e2e40a2229831b2 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Mon, 24 Nov 2025 15:48:21 +0000 Subject: [PATCH 13/52] [accli] E: Launch Applications By Name --- Cargo.lock | 1 + accli/src/main.rs | 14 +++++++++++++- accli/src/tasks/applications.rs | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index d407361..fdd4a30 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4079,6 +4079,7 @@ dependencies = [ [[package]] name = "snpguest" version = "0.10.0" +source = "git+https://github.com/faasm/snpguest.git#cb4dcf19403edd0b3be458f82df7d72c89e6a0e6" dependencies = [ "anyhow", "asn1-rs", diff --git a/accli/src/main.rs b/accli/src/main.rs index 33cca38..e356ef5 100644 --- a/accli/src/main.rs +++ b/accli/src/main.rs @@ -2,7 +2,7 @@ use crate::{ env::Env, tasks::{ accless::Accless, - applications::Applications, + applications::{self, Applications}, attestation_service::AttestationService, azure::Azure, dev::Dev, @@ -344,6 +344,15 @@ enum ApplicationsCommand { #[arg(long)] cert_path: Option, }, + /// Run one of the Accless applications + Run { + /// Type of the application to run + #[arg(long)] + app_type: applications::ApplicationType, + /// Name of the application to run + #[arg(long)] + app_name: applications::Functions, + }, } #[tokio::main] @@ -380,6 +389,9 @@ async fn main() -> anyhow::Result<()> { } => { Applications::build(*clean, *debug, cert_path.as_deref(), false)?; } + ApplicationsCommand::Run { app_type, app_name } => { + Applications::run(app_type.clone(), app_name.clone())?; + } }, Command::Dev { dev_command } => match dev_command { DevCommand::BumpVersion { diff --git a/accli/src/tasks/applications.rs b/accli/src/tasks/applications.rs index cbd6bb6..c413960 100644 --- a/accli/src/tasks/applications.rs +++ b/accli/src/tasks/applications.rs @@ -1,6 +1,18 @@ use crate::tasks::docker::{DOCKER_ACCLESS_CODE_MOUNT_DIR, Docker}; +use clap::ValueEnum; use std::path::Path; +#[derive(Clone, Debug, ValueEnum)] +pub enum ApplicationType { + Function, +} + +#[derive(Clone, Debug, ValueEnum)] +pub enum Functions { + #[value(name = "escrow-xput")] + EscrowXput, +} + #[derive(Debug)] pub struct Applications {} @@ -42,4 +54,25 @@ impl Applications { })?; Docker::run(&cmd, true, Some(workdir_str), &[], false, capture_output) } + + pub fn run(app_type: ApplicationType, app_name: Functions) -> anyhow::Result> { + let binary_path = match app_type { + ApplicationType::Function => { + let binary_name = match app_name { + Functions::EscrowXput => "escrow-xput", + }; + Path::new(DOCKER_ACCLESS_CODE_MOUNT_DIR) + .join("applications/build-native/functions") + .join(binary_name) + .join(binary_name) + } + }; + + let binary_path_str = binary_path.to_str().ok_or_else(|| { + anyhow::anyhow!("Binary path is not valid UTF-8: {}", binary_path.display()) + })?; + let cmd = vec![binary_path_str.to_string()]; + + Docker::run(&cmd, true, None, &[], false, false) + } } From 61a9b9030337a6ca0ea3bdaa0900f3f71d4084eb Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Mon, 24 Nov 2025 17:15:38 +0000 Subject: [PATCH 14/52] [accless] B: Add Warning Around Mock Quote --- accless/libs/attestation/mock.h | 2 +- accless/libs/attestation/snp.cpp | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/accless/libs/attestation/mock.h b/accless/libs/attestation/mock.h index e8fadfa..4cb5ff6 100644 --- a/accless/libs/attestation/mock.h +++ b/accless/libs/attestation/mock.h @@ -10,7 +10,7 @@ namespace accless::attestation::mock { const std::array MOCK_QUOTE_MAGIC_SNP = {'A', 'C', 'C', 'L', 'S', 'N', 'P', '!'}; -const std::string MOCK_GID = "baz"; +const std::string MOCK_GID = "MOCKGID"; const std::string MOCK_WORKFLOW_ID = "foo"; const std::string MOCK_NODE_ID = "bar"; diff --git a/accless/libs/attestation/snp.cpp b/accless/libs/attestation/snp.cpp index 03bbfc0..d6a8cb1 100644 --- a/accless/libs/attestation/snp.cpp +++ b/accless/libs/attestation/snp.cpp @@ -181,6 +181,7 @@ std::vector getReport(std::array reportData) { return getSnpReportFromDev(reportData, std::nullopt); } + // FIXME: enable report data for AzCVM use-case. if (std::filesystem::exists("/dev/tpmrm0")) { return getSnpReportFromTPM(); } @@ -204,7 +205,9 @@ std::string getAttestationJwt(const std::string &gid, // Fetch HW attestation report and include the auxiliary report data in // the signature. std::vector report; + // FIXME: consider making this check more reliable. if (gid == mock::MOCK_GID) { + std::cout << "accless(att): WARNING: mocking SNP quote" << std::endl; report = accless::attestation::mock::buildMockQuote( reportDataVec, mock::MOCK_QUOTE_MAGIC_SNP); } else { From 38d44dc47f01ea938e2fe4a0df36c8bc0b959731 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Mon, 24 Nov 2025 18:33:53 +0000 Subject: [PATCH 15/52] [accli] E: Add Wrappers To Run In cVM --- accli/src/main.rs | 158 +++++++++---- accli/src/tasks/applications.rs | 214 +++++++++++++---- accli/src/tasks/attestation_service.rs | 18 +- accli/src/tasks/cvm.rs | 305 +++++++++++++++++++++++++ accli/src/tasks/mod.rs | 1 + 5 files changed, 605 insertions(+), 91 deletions(-) create mode 100644 accli/src/tasks/cvm.rs diff --git a/accli/src/main.rs b/accli/src/main.rs index e356ef5..03cb5f0 100644 --- a/accli/src/main.rs +++ b/accli/src/main.rs @@ -5,6 +5,7 @@ use crate::{ applications::{self, Applications}, attestation_service::AttestationService, azure::Azure, + cvm::{self, Component, parse_host_guest_path}, dev::Dev, docker::{Docker, DockerContainer}, experiments::{E2eSubScommand, Experiment, UbenchSubCommand}, @@ -12,6 +13,7 @@ use crate::{ }, }; use clap::{Parser, Subcommand}; +use env_logger::Builder; use std::{collections::HashMap, path::PathBuf, process}; pub mod attestation_service; @@ -50,11 +52,6 @@ enum Command { #[command(subcommand)] dev_command: DevCommand, }, - /// Build and push different docker images - Docker { - #[command(subcommand)] - docker_command: DockerCommand, - }, /// Run evaluation experiments and plot results Experiment { #[command(subcommand)] @@ -141,6 +138,40 @@ enum DevCommand { #[arg(long)] force: bool, }, + /// Build and push different docker images + Docker { + #[command(subcommand)] + docker_command: DockerCommand, + }, + /// Build and run a cVM image + Cvm { + #[command(subcommand)] + cvm_command: CvmCommand, + }, +} + +#[derive(Debug, Subcommand)] +enum CvmCommand { + /// Run a command inside the cVM. + Run { + #[arg(last = true)] + cmd: Vec, + /// Optional: SCP files into the cVM. + /// Specify as :. Can be repeated. + #[arg(long, value_name = "HOST_PATH:GUEST_PATH", value_parser = parse_host_guest_path)] + scp_file: Vec<(PathBuf, PathBuf)>, + /// Set the working directory inside the container + #[arg(long)] + cwd: Option, + }, + /// Configure the cVM image by building and installing the different + /// components. + Setup { + #[arg(long)] + clean: bool, + #[arg(long)] + component: Option, + }, } #[derive(Debug, Subcommand)] @@ -343,33 +374,36 @@ enum ApplicationsCommand { /// Path to the attestation service's public certificate PEM file. #[arg(long)] cert_path: Option, + /// Whether to build the application inside a cVM. + #[arg(long, default_value_t = false)] + in_cvm: bool, }, /// Run one of the Accless applications Run { /// Type of the application to run - #[arg(long)] app_type: applications::ApplicationType, /// Name of the application to run - #[arg(long)] app_name: applications::Functions, + /// Whether to run the application inside a cVM. + #[arg(long, default_value_t = false)] + in_cvm: bool, + /// URL of the attestation service to contact. + #[arg(long)] + as_url: Option, + /// Path to the attestation service's public certificate PEM file. + #[arg(long)] + as_cert_path: Option, }, } #[tokio::main] async fn main() -> anyhow::Result<()> { - let cli = Cli::parse(); - - // Initialize the logger based on the debug flag - if cli.debug { - env_logger::Builder::from_default_env() - .filter_level(log::LevelFilter::Debug) - .init(); - } else { - env_logger::Builder::from_default_env() - .filter_level(log::LevelFilter::Info) - .init(); - } + // Initialize the logger. + let env = env_logger::Env::default().filter_or("RUST_LOG", "info"); + let mut builder = Builder::from_env(env); + builder.init(); + let cli = Cli::parse(); match &cli.task { Command::Accless { accless_command } => match accless_command { AcclessCommand::Build { clean, debug } => { @@ -386,11 +420,24 @@ async fn main() -> anyhow::Result<()> { clean, debug, cert_path, + in_cvm, } => { - Applications::build(*clean, *debug, cert_path.as_deref(), false)?; + Applications::build(*clean, *debug, cert_path.as_deref(), false, *in_cvm)?; } - ApplicationsCommand::Run { app_type, app_name } => { - Applications::run(app_type.clone(), app_name.clone())?; + ApplicationsCommand::Run { + app_type, + app_name, + in_cvm, + as_url, + as_cert_path, + } => { + Applications::run( + app_type.clone(), + app_name.clone(), + *in_cvm, + as_url.clone(), + as_cert_path.clone(), + )?; } }, Command::Dev { dev_command } => match dev_command { @@ -412,36 +459,49 @@ async fn main() -> anyhow::Result<()> { DevCommand::Tag { force } => { Dev::tag_code(*force)?; } - }, - Command::Docker { docker_command } => match docker_command { - DockerCommand::Build { ctr, push, nocache } => { - for c in ctr { - Docker::build(c, *push, *nocache); + DevCommand::Docker { docker_command } => match docker_command { + DockerCommand::Build { ctr, push, nocache } => { + for c in ctr { + Docker::build(c, *push, *nocache); + } } - } - DockerCommand::BuildAll { push, nocache } => { - for ctr in DockerContainer::iter_variants() { - // Do not push the base build image - if *ctr == DockerContainer::Experiments { - Docker::build(ctr, false, *nocache); - } else { - Docker::build(ctr, *push, *nocache); + DockerCommand::BuildAll { push, nocache } => { + for ctr in DockerContainer::iter_variants() { + // Do not push the base build image + if *ctr == DockerContainer::Experiments { + Docker::build(ctr, false, *nocache); + } else { + Docker::build(ctr, *push, *nocache); + } } } - } - DockerCommand::Cli { net } => { - Docker::cli(*net)?; - } - DockerCommand::Run { - cmd, - mount, - cwd, - env, - net, - capture_output, - } => { - Docker::run(cmd, *mount, cwd.as_deref(), env, *net, *capture_output)?; - } + DockerCommand::Cli { net } => { + Docker::cli(*net)?; + } + DockerCommand::Run { + cmd, + mount, + cwd, + env, + net, + capture_output, + } => { + Docker::run(cmd, *mount, cwd.as_deref(), env, *net, *capture_output)?; + } + }, + DevCommand::Cvm { cvm_command } => match cvm_command { + CvmCommand::Run { cmd, scp_file, cwd } => { + let scp_files_option = if scp_file.is_empty() { + None + } else { + Some(scp_file.as_slice()) + }; + cvm::run(cmd, scp_files_option, cwd.as_ref())?; + } + CvmCommand::Setup { clean, component } => { + cvm::build(*clean, *component)?; + } + }, }, Command::Experiment { experiments_command: exp, diff --git a/accli/src/tasks/applications.rs b/accli/src/tasks/applications.rs index c413960..ac969a1 100644 --- a/accli/src/tasks/applications.rs +++ b/accli/src/tasks/applications.rs @@ -1,18 +1,64 @@ -use crate::tasks::docker::{DOCKER_ACCLESS_CODE_MOUNT_DIR, Docker}; +use crate::tasks::{ + attestation_service, cvm, + docker::{DOCKER_ACCLESS_CODE_MOUNT_DIR, Docker}, +}; +use anyhow::{Context, Result}; use clap::ValueEnum; -use std::path::Path; +use std::{ + fmt::{Display, Formatter, Result as FmtResult}, + path::{Path, PathBuf}, + str::FromStr, +}; #[derive(Clone, Debug, ValueEnum)] pub enum ApplicationType { Function, } +impl Display for ApplicationType { + fn fmt(&self, f: &mut Formatter) -> FmtResult { + match self { + ApplicationType::Function => write!(f, "function"), + } + } +} + +impl FromStr for ApplicationType { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + match s { + "function" => Ok(ApplicationType::Function), + _ => anyhow::bail!("Invalid ApplicationType: {}", s), + } + } +} + #[derive(Clone, Debug, ValueEnum)] pub enum Functions { #[value(name = "escrow-xput")] EscrowXput, } +impl Display for Functions { + fn fmt(&self, f: &mut Formatter) -> FmtResult { + match self { + Functions::EscrowXput => write!(f, "escrow-xput"), + } + } +} + +impl FromStr for Functions { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + match s { + "escrow-xput" => Ok(Functions::EscrowXput), + _ => anyhow::bail!("Invalid Function: {}", s), + } + } +} + #[derive(Debug)] pub struct Applications {} @@ -22,57 +68,147 @@ impl Applications { debug: bool, cert_path: Option<&str>, capture_output: bool, - ) -> anyhow::Result> { - let mut cmd = vec!["python3".to_string(), "build.py".to_string()]; + in_cvm: bool, + ) -> Result> { + // If --in-cvm flag is passed, we literally re run the same `accli` command, but + // inside the cVM. + let mut cmd = if in_cvm { + vec![ + "./scripts/accli_wrapper.sh".to_string(), + "applications".to_string(), + "build".to_string(), + ] + } else { + vec!["python3".to_string(), "build.py".to_string()] + }; + if clean { cmd.push("--clean".to_string()); } if debug { cmd.push("--debug".to_string()); } - if let Some(cert_path_str) = cert_path { - let cert_path = Path::new(cert_path_str); - if !cert_path.exists() { - anyhow::bail!("Certificate path does not exist: {}", cert_path.display()); + + if in_cvm { + // Make sure the certificates are available in the cVM. + let mut scp_files: Vec<(PathBuf, PathBuf)> = vec![]; + if let Some(cert_path_str) = cert_path { + let host_cert_path = PathBuf::from(cert_path_str); + if !host_cert_path.exists() { + anyhow::bail!( + "Certificate path does not exist: {}", + host_cert_path.display() + ); + } + if !host_cert_path.is_file() { + anyhow::bail!( + "Certificate path is not a file: {}", + host_cert_path.display() + ); + } + let guest_cert_path = PathBuf::from("applications").join( + host_cert_path + .file_name() + .context("Failed to get file name for cert path")?, + ); + scp_files.push((host_cert_path, guest_cert_path.clone())); + + cmd.push("--cert-path".to_string()); + cmd.push(guest_cert_path.display().to_string()); } - if !cert_path.is_file() { - anyhow::bail!("Certificate path is not a file: {}", cert_path.display()); + + cvm::run( + &cmd, + if scp_files.is_empty() { + None + } else { + Some(&scp_files) + }, + None, + )?; + Ok(None) + } else { + if let Some(cert_path_str) = cert_path { + let cert_path = Path::new(cert_path_str); + if !cert_path.exists() { + anyhow::bail!("Certificate path does not exist: {}", cert_path.display()); + } + if !cert_path.is_file() { + anyhow::bail!("Certificate path is not a file: {}", cert_path.display()); + } + let docker_cert_path = Docker::get_docker_path(cert_path)?; + cmd.push("--cert-path".to_string()); + let docker_cert_path_str = docker_cert_path.to_str().ok_or_else(|| { + anyhow::anyhow!( + "Docker path for certificate is not valid UTF-8: {}", + docker_cert_path.display() + ) + })?; + cmd.push(docker_cert_path_str.to_string()); } - let docker_cert_path = Docker::get_docker_path(cert_path)?; - cmd.push("--cert-path".to_string()); - let docker_cert_path_str = docker_cert_path.to_str().ok_or_else(|| { - anyhow::anyhow!( - "Docker path for certificate is not valid UTF-8: {}", - docker_cert_path.display() - ) + let workdir = Path::new(DOCKER_ACCLESS_CODE_MOUNT_DIR).join("applications"); + let workdir_str = workdir.to_str().ok_or_else(|| { + anyhow::anyhow!("Workdir path is not valid UTF-8: {}", workdir.display()) })?; - cmd.push(docker_cert_path_str.to_string()); + Docker::run(&cmd, true, Some(workdir_str), &[], false, capture_output) } - let workdir = Path::new(DOCKER_ACCLESS_CODE_MOUNT_DIR).join("applications"); - let workdir_str = workdir.to_str().ok_or_else(|| { - anyhow::anyhow!("Workdir path is not valid UTF-8: {}", workdir.display()) - })?; - Docker::run(&cmd, true, Some(workdir_str), &[], false, capture_output) } - pub fn run(app_type: ApplicationType, app_name: Functions) -> anyhow::Result> { - let binary_path = match app_type { - ApplicationType::Function => { - let binary_name = match app_name { - Functions::EscrowXput => "escrow-xput", - }; - Path::new(DOCKER_ACCLESS_CODE_MOUNT_DIR) - .join("applications/build-native/functions") - .join(binary_name) - .join(binary_name) - } + pub fn run( + app_type: ApplicationType, + app_name: Functions, + in_cvm: bool, + as_url: Option, + as_cert_path: Option, + ) -> anyhow::Result> { + let binary_name = match app_name { + Functions::EscrowXput => "escrow-xput", }; - let binary_path_str = binary_path.to_str().ok_or_else(|| { - anyhow::anyhow!("Binary path is not valid UTF-8: {}", binary_path.display()) - })?; - let cmd = vec![binary_path_str.to_string()]; + // If --in-cvm flag is passed, we literally re run the same `accli` command, but + // inside the cVM. + if in_cvm { + let cmd = vec![ + "./scripts/accli_wrapper.sh".to_string(), + "applications".to_string(), + "run".to_string(), + format!("{app_type}"), + format!("{app_name}"), + ]; + + // We don't need to SCP any files here, because we assume that the certificates + // have been copied during the build stage, and persisted in the + // disk image. + cvm::run(&cmd, None, None)?; + + Ok(None) + } else { + let binary_path = match app_type { + ApplicationType::Function => Path::new(DOCKER_ACCLESS_CODE_MOUNT_DIR) + .join("applications/build-native") + .join(binary_name) + .join(binary_name), + }; + + let binary_path_str = binary_path.to_str().ok_or_else(|| { + anyhow::anyhow!("Binary path is not valid UTF-8: {}", binary_path.display()) + })?; + let cmd = vec![binary_path_str.to_string()]; + + let as_env_vars: Vec = match (as_url, as_cert_path) { + (Some(as_url), Some(as_cert_path)) => attestation_service::get_as_env_vars( + &as_url, + as_cert_path.to_str().ok_or_else(|| { + anyhow::anyhow!( + "as cert path is not valid UTF-8 (path={})", + as_cert_path.display() + ) + })?, + ), + _ => vec![], + }; - Docker::run(&cmd, true, None, &[], false, false) + Docker::run(&cmd, true, None, &as_env_vars, false, false) + } } } diff --git a/accli/src/tasks/attestation_service.rs b/accli/src/tasks/attestation_service.rs index 7ddf35f..6173436 100644 --- a/accli/src/tasks/attestation_service.rs +++ b/accli/src/tasks/attestation_service.rs @@ -7,6 +7,18 @@ use std::{ process::{Command, Stdio}, }; +const AS_URL_ENV_VAR: &str = "ACCLESS_AS_URL"; +const AS_CERT_PATH_ENV_VAR: &str = "ACCLESS_AS_CERT_PATH"; + +/// Returns the required attestation-service env. vars given a URL and cert +/// path. +pub fn get_as_env_vars(as_url: &str, as_cert_path: &str) -> Vec { + vec![ + format!("{AS_URL_ENV_VAR}={as_url}"), + format!("{AS_CERT_PATH_ENV_VAR}={as_cert_path}"), + ] +} + pub struct AttestationService; impl AttestationService { @@ -67,9 +79,9 @@ impl AttestationService { } pub async fn health(url: Option, cert_path: Option) -> Result<()> { - let url = url.or_else(|| std::env::var("ACCLESS_AS_URL").ok()); + let url = url.or_else(|| std::env::var(AS_URL_ENV_VAR).ok()); let cert_path = cert_path.or_else(|| { - std::env::var("ACCLESS_AS_CERT_PATH") + std::env::var(AS_CERT_PATH_ENV_VAR) .ok() .map(std::path::PathBuf::from) }); @@ -77,7 +89,7 @@ impl AttestationService { let url = match url { Some(url) => url, None => { - anyhow::bail!("Attestation service URL not provided. Set --url or ACCLESS_AS_URL") + anyhow::bail!("Attestation service URL not provided. Set --url or {AS_URL_ENV_VAR}") } }; diff --git a/accli/src/tasks/cvm.rs b/accli/src/tasks/cvm.rs new file mode 100644 index 0000000..9ca9b70 --- /dev/null +++ b/accli/src/tasks/cvm.rs @@ -0,0 +1,305 @@ +//! This module implements helper methods to build and run functions inside a +//! cVM image loaded with Accless' code. + +use crate::env::Env; +use anyhow::{Context, Result}; +use log::{debug, error, info, warn}; +use std::{ + io::{BufRead, BufReader, Read}, + path::PathBuf, + process::{Child, Command, Stdio}, + str::FromStr, + sync::mpsc, + thread, + time::Duration, +}; + +// This timeout needs to be long enough to accomodate for the full VM set-up (in +// the worst case) which involves building all the dependencies inside the cVM. +const CVM_BOOT_TIMEOUT: Duration = Duration::from_secs(240); +const CVM_USER: &str = "ubuntu"; +const CVM_ACCLESS_ROOT: &str = "/home/ubuntu/accless"; +const EPH_PRIVKEY: &str = "snp-key"; +const SSH_PORT: u16 = 2222; + +pub fn parse_host_guest_path(s: &str) -> anyhow::Result<(PathBuf, PathBuf)> { + let parts: Vec<&str> = s.split(':').collect(); + if parts.len() == 2 { + Ok((PathBuf::from(parts[0]), PathBuf::from(parts[1]))) + } else { + anyhow::bail!("Invalid HOST_PATH:GUEST_PATH format: {}", s) + } +} + +#[derive(Debug, Clone, Copy)] +pub enum Component { + Apt, + Qemu, + Ovmf, + Kernel, + Disk, + Keys, + Cloudinit, +} + +impl FromStr for Component { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + match s { + "apt" => Ok(Component::Apt), + "qemu" => Ok(Component::Qemu), + "ovmf" => Ok(Component::Ovmf), + "kernel" => Ok(Component::Kernel), + "disk" => Ok(Component::Disk), + "keys" => Ok(Component::Keys), + "cloudinit" => Ok(Component::Cloudinit), + _ => anyhow::bail!("Invalid component: {}", s), + } + } +} + +impl std::fmt::Display for Component { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Component::Apt => write!(f, "apt"), + Component::Qemu => write!(f, "qemu"), + Component::Ovmf => write!(f, "ovmf"), + Component::Kernel => write!(f, "kernel"), + Component::Disk => write!(f, "disk"), + Component::Keys => write!(f, "keys"), + Component::Cloudinit => write!(f, "cloudinit"), + } + } +} + +struct QemuGuard { + child: Child, +} + +impl Drop for QemuGuard { + fn drop(&mut self) { + info!("Killing QEMU process with PID: {}", self.child.id()); + if let Err(e) = self.child.kill() { + error!("Failed to kill QEMU process: {}", e); + } + } +} + +fn snp_root() -> PathBuf { + let mut path = Env::proj_root(); + path.push("scripts"); + path.push("snp"); + path +} + +fn snp_output_dir() -> PathBuf { + let mut path = snp_root(); + path.push("output"); + path +} + +/// Helper method to read the logs from the cVM's stdout until it is ready. +fn wait_for_cvm_ready(reader: R, timeout: Duration) -> Result<()> { + let mut reader = BufReader::new(reader); + let (tx, rx) = mpsc::channel::<()>(); + + thread::spawn(move || { + let tx = tx; + loop { + let mut line = String::new(); + match reader.read_line(&mut line) { + Ok(0) => { + // EOF + break; + } + Ok(_) => { + let trimmed = line.trim_end(); + // Optional: forward QEMU output to your logs + debug!("wait_for_cvm_ready(): [cVM] {trimmed}"); + + // Look for your ready marker; keep it loose so version etc. don't matter + if trimmed.contains("cloud-init") + && trimmed.contains("Accless SNP test instance") + && trimmed.contains("ready") + { + let _ = tx.send(()); + break; + } + } + Err(e) => { + warn!("Error reading cVM stdout: {e}"); + break; + } + } + } + }); + + match rx.recv_timeout(timeout) { + Ok(()) => Ok(()), + Err(mpsc::RecvTimeoutError::Timeout) => { + anyhow::bail!("timed out waiting for cVM to become ready") + } + Err(e) => anyhow::bail!("cVM stdout reader terminated unexpectedly (error={e})"), + } +} + +pub fn build(clean: bool, component: Option) -> Result<()> { + info!("build(): building cVM image..."); + let mut cmd = Command::new(format!("{}/setup.sh", snp_root().display())); + cmd.current_dir(Env::proj_root()); + + if clean { + cmd.arg("--clean"); + } + + if let Some(component) = component { + cmd.arg("--component").arg(format!("{}", component)); + } + + let status = cmd.status()?; + if !status.success() { + anyhow::bail!("Failed to build cVM image"); + } + + Ok(()) +} + +/// Run a command inside a cVM. +/// +/// Runs a command inside a confidential VM (cVM) after optionally copying files +/// to it. This function first starts the cVM, waits for it to become ready, +/// then SCPs any specified files, and finally executes the given command via +/// SSH. +/// +/// # Arguments +/// +/// - `cmd`: A slice of strings representing the command and its arguments to be +/// executed inside the cVM. +/// - `scp_files`: An optional slice of `(HostPath, GuestPath)` tuples. +/// `HostPath` is the path to the file on the host machine, and `GuestPath` is +/// the relative path inside the cVM. The `GuestPath` will automatically be +/// prefixed with `/home/ubuntu/accless`. +/// +/// # Returns +/// +/// A `Result` indicating success or failure. +/// +/// # Example Usage +/// +/// ``` +/// use anyhow::Result; +/// use std::path::PathBuf; +/// +/// // Example of running a command without SCPing files +/// cvm::run(&["ls".to_string(), "-la".to_string()], None).unwrap(); +/// +/// // Example of SCPing a file and then running a command +/// let host_path = PathBuf::from("./my_local_file.txt"); +/// let guest_path = PathBuf::from("my_remote_file.txt"); +/// cvm::run( +/// &["cat".to_string(), "my_remote_file.txt".to_string()], +/// Some(&[(host_path, guest_path)]), +/// ) +/// .unwrap(); +/// ``` +/// - `cwd`: An optional `PathBuf` representing the working directory inside the +/// cVM, relative to `/home/ubuntu/accless`. If provided, the command will be +/// executed in this directory. +pub fn run( + cmd: &[String], + scp_files: Option<&[(PathBuf, PathBuf)]>, + cwd: Option<&PathBuf>, +) -> Result<()> { + // Start QEMU and capture stdout. + info!("run(): starting cVM..."); + let mut qemu_child = Command::new(format!("{}/run.sh", snp_root().display())) + .current_dir(Env::proj_root()) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::inherit()) + .spawn() + .context("run(): failed to spawn QEMU")?; + + let qemu_stdout = qemu_child + .stdout + .take() + .context("run(): failed to capture QEMU stdout")?; + + let _qemu_guard = QemuGuard { child: qemu_child }; + + // Wait for cloud-init to signal readiness. + info!("run(): waiting for cVM to become ready"); + wait_for_cvm_ready(qemu_stdout, CVM_BOOT_TIMEOUT)?; + + // SCP files into the cVM, if any. + if let Some(files) = scp_files { + info!("run(): copying files into cVM..."); + for (host_path, guest_path) in files { + let full_guest_path = format!("{}/{}", CVM_ACCLESS_ROOT, guest_path.display()); + info!( + "run(): copying {} to {CVM_USER}@localhost:{}", + host_path.display(), + full_guest_path + ); + + let mut scp_cmd = Command::new("scp"); + scp_cmd + .arg("-P") + .arg(SSH_PORT.to_string()) + .arg("-i") + .arg(format!("{}/{EPH_PRIVKEY}", snp_output_dir().display())) + .arg("-o") + .arg("StrictHostKeyChecking=no") + .arg("-o") + .arg("UserKnownHostsFile=/dev/null") + .arg(host_path) + .arg(format!("{CVM_USER}@localhost:{}", full_guest_path)); + + let status = scp_cmd.status()?; + if !status.success() { + anyhow::bail!( + "run(): failed to copy file {} to cVM (exit_code={})", + host_path.display(), + status.code().unwrap_or_default() + ); + } + } + } + + // Construct the command to run in the cVM, including `cd` if `cwd` is + // specified. + let mut final_cmd: Vec = Vec::new(); + final_cmd.push("cd".to_string()); + if let Some(cwd_path) = cwd { + final_cmd.push(format!("{}/{}", CVM_ACCLESS_ROOT, cwd_path.display())); + } else { + final_cmd.push(CVM_ACCLESS_ROOT.to_string()); + } + final_cmd.push("&&".to_string()); + final_cmd.extend_from_slice(cmd); + + info!( + "run(): running command in cVM (cmd='{}')", + final_cmd.join(" ") + ); + let mut ssh_cmd = Command::new("ssh"); + ssh_cmd + .arg("-p") + .arg(SSH_PORT.to_string()) + .arg("-i") + .arg(format!("{}/{EPH_PRIVKEY}", snp_output_dir().display())) + .arg("-o") + .arg("StrictHostKeyChecking=no") + .arg("-o") + .arg("UserKnownHostsFile=/dev/null") + .arg(format!("{CVM_USER}@localhost")) + .args(final_cmd); + + let status = ssh_cmd.status()?; + if !status.success() { + anyhow::bail!("run(): command failed to execute in cVM"); + } + + Ok(()) +} diff --git a/accli/src/tasks/mod.rs b/accli/src/tasks/mod.rs index ef7bf82..bdf2ebd 100644 --- a/accli/src/tasks/mod.rs +++ b/accli/src/tasks/mod.rs @@ -2,6 +2,7 @@ pub mod accless; pub mod applications; pub mod attestation_service; pub mod azure; +pub mod cvm; pub mod dev; pub mod docker; pub mod experiments; From 1c96ecf531971eba372aa9e7f5286d608c7d96a4 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Mon, 24 Nov 2025 18:45:24 +0000 Subject: [PATCH 16/52] [scripts] B: Make accli_wrapper.sh Resilient --- scripts/accli_wrapper.sh | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/scripts/accli_wrapper.sh b/scripts/accli_wrapper.sh index 3e930ca..2a64a6c 100755 --- a/scripts/accli_wrapper.sh +++ b/scripts/accli_wrapper.sh @@ -3,6 +3,23 @@ set -e source ./scripts/workon.sh +# Check if cargo is available +if ! command -v cargo &> /dev/null; then + echo "cargo not found in PATH. Attempting to source $HOME/.cargo/env" + if [ -f "$HOME/.cargo/env" ]; then + source "$HOME/.cargo/env" + else + echo "Error: $HOME/.cargo/env not found. Please install Rust or ensure it's correctly configured." + exit 1 + fi + + # Check again after sourcing + if ! command -v cargo &> /dev/null; then + echo "Error: cargo still not found after sourcing $HOME/.cargo/env. Please ensure Rust is installed and configured correctly." + exit 1 + fi +fi + THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]:-${(%):-%x}}" )" >/dev/null 2>&1 && pwd )" PROJ_ROOT="${THIS_DIR}/.." RUST_ROOT="${PROJ_ROOT}/invrs" From ff16d9c584eb5cd17e7934fbe327d4aece49e4ba Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Tue, 25 Nov 2025 14:36:50 +0000 Subject: [PATCH 17/52] [scripts] E: Add Offline Disk Provisioning for SNP --- scripts/snp/cloud-init/meta-data.in | 2 +- scripts/snp/cloud-init/user-data.in | 94 ++++++---------- scripts/snp/run.sh | 2 +- scripts/snp/setup.sh | 159 +++++++++++++++++++++++++++- 4 files changed, 192 insertions(+), 65 deletions(-) diff --git a/scripts/snp/cloud-init/meta-data.in b/scripts/snp/cloud-init/meta-data.in index 5e212f0..4325a24 100644 --- a/scripts/snp/cloud-init/meta-data.in +++ b/scripts/snp/cloud-init/meta-data.in @@ -1,2 +1,2 @@ -instance-id: ${INSTANCE_ID} +instance-id: accless-snp-dev local-hostname: accless-snp diff --git a/scripts/snp/cloud-init/user-data.in b/scripts/snp/cloud-init/user-data.in index 632790b..83032f0 100644 --- a/scripts/snp/cloud-init/user-data.in +++ b/scripts/snp/cloud-init/user-data.in @@ -1,69 +1,43 @@ #cloud-config - +# +# This cloud-init config is intentionally minimal. +# Most provisioning (packages, users, rust, git clone, etc.) +# is done offline via scripts/snp/setup.sh using qemu-nbd. +# +# We assume the 'ubuntu' user already exists in the image +# with UID 2000 and appropriate groups (sudo, docker, sevguest). +# Cloud-init just injects the SSH key. +# +# If you modify this file, you most likely will need to recreate the cVM's disk +# image from scratch. To do this, you may run: +# +# accli dev cvm build --component disk +# accli dev cvm build --component cloudinit +# +# The next time you run a cVM, it will re-run all the comands specified herein. + +# Configure user `ubuntu`. users: - name: ubuntu - gecos: Ubuntu - sudo: ALL=(ALL) NOPASSWD:ALL - groups: [sudo, docker] - shell: /bin/bash - ssh_authorized_keys: + ssh-authorized-keys: - "${SSH_PUB_KEY}" -growpart: - mode: auto - devices: ['/'] - ignore_growroot_disabled: false - -package_update: true -packages: - - docker.io - - git - - linux-modules-extra-${GUEST_KERNEL_VERSION} - - python3-venv - +# Very light runtime tweaks only. runcmd: - # Make sure ubuntu is in docker group (in case group created after user). - - [ sh, -c, 'usermod -aG docker ubuntu || true' ] - # Enable and start docker. + # Make sure /dev/sev-guest udev rule has taken effect and module is loaded. + - [ sh, -c, ' + groupadd -r sevguest || true; + usermod -aG sevguest ubuntu || true; + modprobe sev-guest || true; + udevadm control --reload-rules; + udevadm trigger /dev/sev-guest || true; + ' ] + # Ensure docker + sshd are running (systemctl enable handled offline already). - [ systemctl, enable, --now, docker ] - # Enable and start sshd. - [ systemctl, enable, --now, ssh.service ] - # Load SEV guest driver so /dev/sev-guest exists. - - [ sh, -c, 'modprobe sev-guest || echo "modprobe sev-guest failed (maybe built-in?)"' ] - # Change password. - - [ sh, -c, 'echo "ubuntu:ubuntu" | chpasswd' ] - - [ bash, -c, ' - if [ "$(id -u ubuntu)" -eq 1000 ]; then - echo "[cloud-init] Changing ubuntu UID/GID from 1000 to 2000"; - - # Change group first - groupmod -g 2000 ubuntu; - - # Change user UID - usermod -u 2000 ubuntu; - - # Fix ownership of files that still belong to uid/gid 1000 - # Restrict to typical places to avoid scanning the whole fs if you prefer - for path in /home /var /etc; do - find "$path" -xdev -uid 1000 -exec chown -h 2000:2000 {} \; || true - find "$path" -xdev -gid 1000 -exec chgrp -h 2000 {} \; || true - done - - echo "[cloud-init] ubuntu is now: $(id ubuntu)"; - else - echo "[cloud-init] ubuntu UID is $(id -u ubuntu), not 1000; skipping UID change."; - fi - ' ] - # Make sure home is owned by ubuntu> - - [ chown, "-R", "ubuntu:ubuntu", "/home/ubuntu" ] - # Install rust. - - [ sudo, "-u", "ubuntu", "bash", "-lc", "curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y" ] - # Clone Accless repo. - # FIXME: remove branch name. - - [ sudo, "-u", "ubuntu", "bash", "-lc", "cd /home/ubuntu && git clone -b feature-escrow-func https://github.com/faasm/tless.git accless" ] - # Fetch Accless docker image. - - [ sudo, "-u", "ubuntu", "bash", "-lc", "docker pull ghcr.io/faasm/accless-experiments:${ACCLESS_VERSION}" ] - # Quick debug markers in console. - - [ sh, -c, 'echo "[cloud-init] Docker + chpasswd + sev-guest setup done"' ] + # Optional: pull docker image at runtime (idempotent). + - [ sudo, "-u", "ubuntu", "bash", "-lc", "docker pull ghcr.io/faasm/accless-experiments:${ACCLESS_VERSION} || true" ] + # Explicit readiness marker on the serial console (for your Rust QEMU wrapper). + - [ sh, -c, 'logger -t cloud-init "Accless SNP test instance v${ACCLESS_VERSION} is ready."' ] -final_message: "Accless SNP test instance v2 is ready." +final_message: "Accless SNP test instance v${ACCLESS_VERSION} is ready." diff --git a/scripts/snp/run.sh b/scripts/snp/run.sh index d19292a..2362bf6 100755 --- a/scripts/snp/run.sh +++ b/scripts/snp/run.sh @@ -36,7 +36,7 @@ run_qemu() { # Can SSH into the VM witih: # ssh -p 2222 -i ${OUTPUT_DIR}/snp-key ubuntu@localhost - ${qemu} \ + exec ${qemu} \ -L "${qemu_bios_dir}" \ -enable-kvm \ -nographic \ diff --git a/scripts/snp/setup.sh b/scripts/snp/setup.sh index b53a652..7825a24 100755 --- a/scripts/snp/setup.sh +++ b/scripts/snp/setup.sh @@ -101,7 +101,159 @@ fetch_kernel() { } # -# Fetch the linux kernel image. +# Provision the disk image. +# +provision_disk_image() { + local disk_img="${OUTPUT_DIR}/disk.img" + print_info "Provisioning disk image (path=${disk_img})..." + + local root_mnt="/mnt/cvm-root" + local qemu_nbd="${OUTPUT_DIR}/qemu/qemu-${QEMU_VERSION}/build/qemu-nbd" + + # Attach to disk image. + sudo modprobe nbd max_part=8 + sudo ${qemu_nbd} --connect=/dev/nbd0 "${disk_img}" + sudo partprobe /dev/nbd0 2>/dev/null || true + + # Fix GPT metadata after qemu-img resize. + sudo sgdisk -e /dev/nbd0 + + disk_provision_cleanup() { + local root_mnt=$1 + local qemu_nbd=$2 + echo "[provision-disk] Cleaning up..." + set +e + sudo umount -R "${root_mnt}" 2>/dev/null || true + sudo ${qemu_nbd} --disconnect /dev/nbd0 2>/dev/null || true + sudo rm -rf "${root_mnt}" + } + trap "disk_provision_cleanup '${root_mnt}' '${qemu_nbd}'" EXIT + + local root_dev="/dev/nbd0p1" + + # Grow the ext4 filesystem to occupy all the disk space. + print_info "[provision-disk] Growing filesystem..." + sudo parted /dev/nbd0 --script resizepart 1 100% + sudo e2fsck -f ${root_dev} # > /dev/null 2>&1 + sudo resize2fs ${root_dev} # > /dev/null 2>&1 + + print_info "[provision-disk] Mounting root filesystem ${root_dev} at ${root_mnt}..." + sudo mkdir -p "${root_mnt}" + sudo mount "${root_dev}" "${root_mnt}" + + # Make sure DNS works inside chroot. + sudo install -m 644 /etc/resolv.conf "${root_mnt}/etc/resolv.conf" + + print_info "[provision-disk] Bind-mounting /dev /proc /sys /run..." + for d in dev proc sys run; do + sudo mount --bind "/${d}" "${root_mnt}/${d}" + done + + print_info "[provision] Running provisioning commands inside chroot..." + sudo GUEST_KERNEL_VERSION=${GUEST_KERNEL_VERSION} chroot "${root_mnt}" /bin/bash <<'EOF' +set -euo pipefail + +echo "[provision/chroot] Starting..." + +# Force IPv4 in the chroot. +echo 'Acquire::ForceIPv4 "true";' > /etc/apt/apt.conf.d/99force-ipv4 + +export DEBIAN_FRONTEND=noninteractive + +echo "[provision/chroot] apt-get update & base packages..." +apt-get update > /dev/null 2>&1 +apt-get install -y \ + build-essential \ + ca-certificates \ + curl \ + docker.io \ + git \ + libfontconfig1-dev \ + libssl-dev \ + "linux-modules-extra-${GUEST_KERNEL_VERSION}" \ + python3-venv \ + pkg-config \ + sudo > /dev/null 2>&1 + +# Run depmod to re-configure kernel modules. +depmod -a "${GUEST_KERNEL_VERSION}" + +# Ensure ubuntu user exists and adjust UID to 2000 if needed +if id ubuntu >/dev/null 2>&1; then + U_OLD_UID="$(id -u ubuntu)" + if [ "${U_OLD_UID}" -eq 1000 ]; then + echo "[provision/chroot] Changing ubuntu UID/GID from 1000 to 2000..." + # Make sure 2000 is free + if getent passwd 2000 >/dev/null; then + echo "[provision/chroot] UID 2000 already in use, aborting UID change" >&2 + exit 1 + fi + if getent group 2000 >/dev/null; then + echo "[provision/chroot] GID 2000 already in use, aborting GID change" >&2 + exit 1 + fi + groupmod -g 2000 ubuntu + usermod -u 2000 ubuntu + + echo "[provision/chroot] Fixing ownership for UID/GID 1000..." + for path in /home /var /etc; do + find "$path" -xdev -uid 1000 -exec chown -h 2000 {} \; || true + find "$path" -xdev -gid 1000 -exec chgrp -h 2000 {} \; || true + done + else + echo "[provision/chroot] ubuntu UID is ${U_OLD_UID}, leaving as-is." + fi +else + echo "[provision/chroot] ubuntu user not found, creating with UID 2000..." + groupadd -g 2000 ubuntu || true + useradd -m -u 2000 -g 2000 -s /bin/bash ubuntu +fi + +echo "[provision/chroot] Ensuring groups docker, sevguest, sudo memberships..." +groupadd -r docker || true +groupadd -r sevguest || true +usermod -aG sudo,docker,sevguest ubuntu || true + +# Allow ubuntu user to use sudo without a password. +echo "ubuntu ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/ubuntu-nopasswd +chmod 440 /etc/sudoers.d/ubuntu-nopasswd + +echo "[provision/chroot] Writing /etc/udev/rules.d/90-sev-guest.rules..." +cat >/etc/udev/rules.d/90-sev-guest.rules <<'RULE' +KERNEL=="sev-guest", GROUP="sevguest", MODE="0660" +RULE + +echo "[provision/chroot] Enabling docker & sshd..." +# systemctl enable will just manipulate symlinks; OK even if systemd not running +systemctl enable docker > /dev/null 2>&1 || true +systemctl enable ssh > /dev/null 2>&1 || systemctl enable ssh.service > /dev/null 2>&1 || true + +echo "[provision/chroot] Installing rustup for ubuntu..." +su -l ubuntu -c 'curl --proto "=https" --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y > /dev/null 2>&1' || true + +# FIXME: remove branch name and build some deps. +echo "[provision/chroot] Cloning Accless repo (idempotent)..." +su -l ubuntu -c ' + cd /home/ubuntu && + if [ ! -d accless/.git ]; then + git clone -b feature-escrow-func https://github.com/faasm/tless.git accless > /dev/null 2>&1; + else + echo "accless repo already present, skipping clone"; + fi +' || true + +# (Optional) You *could* pull docker images here, but it's messy because dockerd +# isn't running inside the chroot. Leaving that for runtime (cloud-init or first use). + +echo "[provision/chroot] Provisioning done." +EOF + + disk_provision_cleanup ${root_mnt} ${qemu_nbd} + print_success "Done provisioning disk image!" +} + +# +# Fetch the disk image. # fetch_disk_image() { print_info "Fetching cloud-init disk image..." @@ -118,6 +270,8 @@ fetch_disk_image() { local qemu_img="${OUTPUT_DIR}/qemu/qemu-${QEMU_VERSION}/build/qemu-img" ${qemu_img} resize "${OUTPUT_DIR}/disk.img" +20G > /dev/null 2>&1 print_success "cloud-init disk image resized successfully." + + provision_disk_image } # @@ -139,8 +293,7 @@ prepare_cloudinit_image() { local out_dir="${OUTPUT_DIR}/cloud-init" mkdir -p ${out_dir} - INSTANCE_ID="accless-snp-$(date +%s)" envsubst '${INSTANCE_ID}' \ - < ${in_dir}/meta-data.in > ${out_dir}/meta-data + cp ${in_dir}/meta-data.in ${out_dir}/meta-data ACCLESS_VERSION=$(cat "${ROOT_DIR}/VERSION") \ GUEST_KERNEL_VERSION=${GUEST_KERNEL_VERSION} \ SSH_PUB_KEY=$(cat "${OUTPUT_DIR}/snp-key.pub") \ From d88e034e26c09208bd1dc7246b66845a34e5b070 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Tue, 25 Nov 2025 14:37:09 +0000 Subject: [PATCH 18/52] [scripts] B: Do Not Use Sudo On Workon.sh --- scripts/workon.sh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/scripts/workon.sh b/scripts/workon.sh index a45a61d..e52b9e8 100755 --- a/scripts/workon.sh +++ b/scripts/workon.sh @@ -43,12 +43,6 @@ alias kubectl=${COCO_SOURCE}/bin/kubectl export FAASM_INI_FILE=/home/tless/git/faasm/faasm/faasm.ini export FAASM_VERSION=0.33.0 -# ---------------------------- -# APT deps -# ---------------------------- - -source ${THIS_DIR}/apt.sh - # ---------------------------- # Python deps # ---------------------------- From ac6c3e4c77975a46e53ffc3c1aadc46db71eeac9 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Tue, 25 Nov 2025 14:38:02 +0000 Subject: [PATCH 19/52] [accli] E: Add 'dev cvm cli' --- accli/src/main.rs | 9 ++++++++ accli/src/tasks/cvm.rs | 52 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/accli/src/main.rs b/accli/src/main.rs index 03cb5f0..8baa6b3 100644 --- a/accli/src/main.rs +++ b/accli/src/main.rs @@ -172,6 +172,12 @@ enum CvmCommand { #[arg(long)] component: Option, }, + /// Get an interactive shell inside the cVM. + Cli { + /// Set the working directory inside the container + #[arg(long)] + cwd: Option, + }, } #[derive(Debug, Subcommand)] @@ -501,6 +507,9 @@ async fn main() -> anyhow::Result<()> { CvmCommand::Setup { clean, component } => { cvm::build(*clean, *component)?; } + CvmCommand::Cli { cwd } => { + cvm::cli(cwd.as_ref())?; + } }, }, Command::Experiment { diff --git a/accli/src/tasks/cvm.rs b/accli/src/tasks/cvm.rs index 9ca9b70..1209da8 100644 --- a/accli/src/tasks/cvm.rs +++ b/accli/src/tasks/cvm.rs @@ -303,3 +303,55 @@ pub fn run( Ok(()) } + +pub fn cli(cwd: Option<&PathBuf>) -> Result<()> { + info!("cli(): starting cVM and opening interactive shell..."); + + let mut qemu_child = Command::new(format!("{}/run.sh", snp_root().display())) + .current_dir(Env::proj_root()) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::inherit()) + .spawn() + .context("cli(): failed to spawn QEMU")?; + + let qemu_stdout = qemu_child + .stdout + .take() + .context("cli(): failed to capture QEMU stdout")?; + + let _qemu_guard = QemuGuard { child: qemu_child }; + + info!("cli(): waiting for cVM to become ready"); + wait_for_cvm_ready(qemu_stdout, CVM_BOOT_TIMEOUT)?; + + // Construct the command to run in the cVM (cd into cwd if provided). + let mut interactive_cmd: Vec = Vec::new(); + if let Some(cwd_path) = cwd { + interactive_cmd.push("cd".to_string()); + interactive_cmd.push(format!("{}/{}", CVM_ACCLESS_ROOT, cwd_path.display())); + interactive_cmd.push("&&".to_string()); + } + interactive_cmd.push("bash".to_string()); // Start a bash shell + + info!("cli(): opening interactive SSH session to cVM"); + let status = Command::new("ssh") + .arg("-p") + .arg(SSH_PORT.to_string()) + .arg("-i") + .arg(format!("{}/{EPH_PRIVKEY}", snp_output_dir().display())) + .arg("-o") + .arg("StrictHostKeyChecking=no") + .arg("-o") + .arg("UserKnownHostsFile=/dev/null") + .arg("-t") // Allocate a pseudo-terminal + .arg(format!("{CVM_USER}@localhost")) + .args(interactive_cmd) + .status()?; + + if !status.success() { + anyhow::bail!("cli(): interactive SSH session failed"); + } + + Ok(()) +} From 20e87e77b47bb18044e27d6b8675876d770d30bd Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Tue, 25 Nov 2025 15:03:24 +0000 Subject: [PATCH 20/52] [accli] B: Fix Application Path --- accli/src/main.rs | 4 ++-- accli/src/tasks/applications.rs | 11 +++++------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/accli/src/main.rs b/accli/src/main.rs index 8baa6b3..b3d2c71 100644 --- a/accli/src/main.rs +++ b/accli/src/main.rs @@ -138,12 +138,12 @@ enum DevCommand { #[arg(long)] force: bool, }, - /// Build and push different docker images + /// Build and run commands in the work-on container image Docker { #[command(subcommand)] docker_command: DockerCommand, }, - /// Build and run a cVM image + /// Build and run commands in an SNP-enabled cVM (requires SNP hardware) Cvm { #[command(subcommand)] cvm_command: CvmCommand, diff --git a/accli/src/tasks/applications.rs b/accli/src/tasks/applications.rs index ac969a1..54a6090 100644 --- a/accli/src/tasks/applications.rs +++ b/accli/src/tasks/applications.rs @@ -161,10 +161,6 @@ impl Applications { as_url: Option, as_cert_path: Option, ) -> anyhow::Result> { - let binary_name = match app_name { - Functions::EscrowXput => "escrow-xput", - }; - // If --in-cvm flag is passed, we literally re run the same `accli` command, but // inside the cVM. if in_cvm { @@ -183,11 +179,14 @@ impl Applications { Ok(None) } else { + // Path matches CMake build directory: + // ./applications/build-natie/{functions,test,workflows}/{name}/{binary_name} let binary_path = match app_type { ApplicationType::Function => Path::new(DOCKER_ACCLESS_CODE_MOUNT_DIR) .join("applications/build-native") - .join(binary_name) - .join(binary_name), + .join(format!("{app_type}")) + .join(format!("{app_name}")) + .join(format!("{app_name}")) }; let binary_path_str = binary_path.to_str().ok_or_else(|| { From 7a3c30abd13da60763f82c26dd12ed9cfe76016f Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Tue, 25 Nov 2025 15:07:00 +0000 Subject: [PATCH 21/52] [accli] B: Make Plural Of Function/Workflow --- accli/src/tasks/applications.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/accli/src/tasks/applications.rs b/accli/src/tasks/applications.rs index 54a6090..4ffedca 100644 --- a/accli/src/tasks/applications.rs +++ b/accli/src/tasks/applications.rs @@ -184,7 +184,8 @@ impl Applications { let binary_path = match app_type { ApplicationType::Function => Path::new(DOCKER_ACCLESS_CODE_MOUNT_DIR) .join("applications/build-native") - .join(format!("{app_type}")) + // Note that we need to make the plural. + .join(format!("{app_type}s")) .join(format!("{app_name}")) .join(format!("{app_name}")) }; From 4f19bb95fa8f11e7fbec6e6173cb0f8e35e4ff50 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Tue, 25 Nov 2025 15:14:18 +0000 Subject: [PATCH 22/52] [scripts] E: Refine SNP Scripts --- scripts/snp/cloud-init/user-data.in | 1 + scripts/snp/run.sh | 7 ++++--- scripts/snp/setup.sh | 10 +++++----- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/scripts/snp/cloud-init/user-data.in b/scripts/snp/cloud-init/user-data.in index 83032f0..0cb9871 100644 --- a/scripts/snp/cloud-init/user-data.in +++ b/scripts/snp/cloud-init/user-data.in @@ -19,6 +19,7 @@ # Configure user `ubuntu`. users: - name: ubuntu + sudo: ALL=(ALL) NOPASSWD:ALL ssh-authorized-keys: - "${SSH_PUB_KEY}" diff --git a/scripts/snp/run.sh b/scripts/snp/run.sh index 2362bf6..d79b16f 100755 --- a/scripts/snp/run.sh +++ b/scripts/snp/run.sh @@ -13,9 +13,9 @@ source "${THIS_DIR}/versions.sh" # Helper method to get the C-bit position directly from hardware. # get_cbitpos() { - modprobe cpuid - local ebx=$(sudo dd if=/dev/cpu/0/cpuid ibs=16 count=32 skip=134217728 2> /dev/null | tail -c 16 | od -An -t u4 -j 4 -N 4 | sed -re 's|^ *||') - local cbitpos=$((ebx & 0x3f)) + modprobe cpuid + local ebx=$(sudo dd if=/dev/cpu/0/cpuid ibs=16 count=32 skip=134217728 2> /dev/null | tail -c 16 | od -An -t u4 -j 4 -N 4 | sed -re 's|^ *||') + local cbitpos=$((ebx & 0x3f)) echo $cbitpos } @@ -36,6 +36,7 @@ run_qemu() { # Can SSH into the VM witih: # ssh -p 2222 -i ${OUTPUT_DIR}/snp-key ubuntu@localhost + # Login: ubuntu:ubuntu exec ${qemu} \ -L "${qemu_bios_dir}" \ -enable-kvm \ diff --git a/scripts/snp/setup.sh b/scripts/snp/setup.sh index 7825a24..395f05c 100755 --- a/scripts/snp/setup.sh +++ b/scripts/snp/setup.sh @@ -123,8 +123,8 @@ provision_disk_image() { local qemu_nbd=$2 echo "[provision-disk] Cleaning up..." set +e - sudo umount -R "${root_mnt}" 2>/dev/null || true - sudo ${qemu_nbd} --disconnect /dev/nbd0 2>/dev/null || true + sudo umount -R "${root_mnt}" > /dev/null 2>&1 || true + sudo ${qemu_nbd} --disconnect /dev/nbd0 > /dev/null 2>&1 || true sudo rm -rf "${root_mnt}" } trap "disk_provision_cleanup '${root_mnt}' '${qemu_nbd}'" EXIT @@ -150,7 +150,7 @@ provision_disk_image() { done print_info "[provision] Running provisioning commands inside chroot..." - sudo GUEST_KERNEL_VERSION=${GUEST_KERNEL_VERSION} chroot "${root_mnt}" /bin/bash <<'EOF' + sudo GUEST_KERNEL_VERSION=${GUEST_KERNEL_VERSION} LC_ALL=C LANG=C chroot "${root_mnt}" /bin/bash <<'EOF' set -euo pipefail echo "[provision/chroot] Starting..." @@ -210,8 +210,8 @@ else fi echo "[provision/chroot] Ensuring groups docker, sevguest, sudo memberships..." -groupadd -r docker || true -groupadd -r sevguest || true +groupadd -r docker > /dev/null 2>&1 || true +groupadd -r sevguest >/dev/null 2>&1 || true usermod -aG sudo,docker,sevguest ubuntu || true # Allow ubuntu user to use sudo without a password. From 6cd2249bc02e0c0ecf01920fddde5d0ed99dae5e Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Tue, 25 Nov 2025 15:43:19 +0000 Subject: [PATCH 23/52] [accli] E: Pass SEV_GID If /dev/sev-guest Present --- accli/src/tasks/docker.rs | 19 +++++++++++++++++++ scripts/docker-entrypoint.sh | 19 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/accli/src/tasks/docker.rs b/accli/src/tasks/docker.rs index fb26d44..2ab789a 100644 --- a/accli/src/tasks/docker.rs +++ b/accli/src/tasks/docker.rs @@ -3,6 +3,7 @@ use clap::ValueEnum; use log::error; use std::{ fmt, + os::unix::fs::MetadataExt, path::{Path, PathBuf}, process::{Command, Stdio}, str::FromStr, @@ -192,6 +193,18 @@ impl Docker { String::from_utf8_lossy(&gid.stdout).trim().to_string() } + /// Helper method to get the group ID of the /dev/sev-guest device. + /// + /// This method is only used when using `accli` inside a cVM. In our cVM set-up we configure + /// /dev/sev-guest to be in a shared group with our user, to avoid having to use `sudo` to run + /// our functions. + fn get_sevguest_group_id() -> Option { + match std::fs::metadata("/dev/sev-guest") { + Ok(metadata) => Some(metadata.gid()), + Err(_) => None, + } + } + fn exec_cmd( cmd: &[String], cwd: Option<&str>, @@ -260,6 +273,12 @@ impl Docker { .arg("-e") .arg(format!("HOST_GID={}", Self::get_group_id())); + if let Some(sevgest_gid) = Self::get_sevguest_group_id() { + run_cmd + .arg("-e") + .arg(format!("SEV_GID={}", sevgest_gid)); + } + for e in env { run_cmd.arg("-e").arg(e); } diff --git a/scripts/docker-entrypoint.sh b/scripts/docker-entrypoint.sh index 09f0585..27984ee 100755 --- a/scripts/docker-entrypoint.sh +++ b/scripts/docker-entrypoint.sh @@ -1,9 +1,13 @@ #!/bin/bash + set -e USER_ID=${HOST_UID:-9001} GROUP_ID=${HOST_GID:-9001} +# Group ID that owns /dev/sev-guest for SNP deployments (if present). +SEV_GID=${SEV_GID:-} + groupadd -g $GROUP_ID accless useradd -u $USER_ID -g $GROUP_ID -s /bin/bash -K UID_MAX=200000 accless usermod -aG sudo accless @@ -19,6 +23,21 @@ chown -R accless:accless ${HOME} echo ". /code/accless/scripts/workon.sh" >> ${HOME}/.bashrc echo ". ${HOME}/.cargo/env" >> ${HOME}/.bashrc +# Add /dev/sev-guest owning group if necessary. +if [ -e /dev/sev-guest ]; then + if [ -n "$SEV_GID" ]; then + # Create a group with that GID if needed (name "sevguest" or whatever) + if ! getent group "$SEV_GID" >/dev/null; then + groupadd -g "$SEV_GID" sevguest || true + fi + + # Add accless to that group (by GID to be robust to name differences) + usermod -aG "$SEV_GID" accless || true + else + echo "WARNING: /dev/sev-guest present but SEV_GID not set!" + fi +fi + exec /usr/sbin/gosu accless bash -lc \ 'source /code/accless/scripts/workon.sh; source "$HOME/.cargo/env"; exec "$@"' \ bash "$@" From be7811bb967564d57adb0c25541820c9e81825d5 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Tue, 25 Nov 2025 17:00:20 +0000 Subject: [PATCH 24/52] [attestation-service] E: Print Info On Start --- .gitignore | 7 +++++-- Cargo.lock | 27 +++++++++++++++++++++++---- Cargo.toml | 1 + attestation-service/bin/gen_keys.sh | 25 ------------------------- attestation-service/src/main.rs | 28 +++++++++++++++++++++++----- attestation-service/src/state.rs | 4 ++++ attestation-service/src/tls.rs | 1 + 7 files changed, 57 insertions(+), 36 deletions(-) delete mode 100755 attestation-service/bin/gen_keys.sh diff --git a/.gitignore b/.gitignore index 159a377..2e6ee87 100644 --- a/.gitignore +++ b/.gitignore @@ -16,5 +16,8 @@ datasets* agent-plans # Default path for auto-generated TLS certificates. -config/certs -config/test-certs +config/attestation-service/certs +config/attestation-service/test-certs + +# Path for attestation service PID. +config/attestation-service/PID diff --git a/Cargo.lock b/Cargo.lock index fdd4a30..ba41eff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -101,6 +101,7 @@ dependencies = [ "indicatif", "log", "minio", + "nix 0.28.0", "plotters", "rand 0.8.5", "regex", @@ -704,7 +705,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad8646f98db542e39fc66e68a20b2144f6a732636df7c2354e74645faaa433ce" dependencies = [ "borsh-derive", - "cfg_aliases", + "cfg_aliases 0.2.1", ] [[package]] @@ -777,6 +778,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cfg_aliases" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" + [[package]] name = "cfg_aliases" version = "0.2.1" @@ -2769,6 +2776,18 @@ dependencies = [ "memoffset", ] +[[package]] +name = "nix" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" +dependencies = [ + "bitflags 2.10.0", + "cfg-if", + "cfg_aliases 0.1.1", + "libc", +] + [[package]] name = "nom" version = "7.1.3" @@ -3299,7 +3318,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" dependencies = [ "bytes", - "cfg_aliases", + "cfg_aliases 0.2.1", "pin-project-lite", "quinn-proto", "quinn-udp", @@ -3339,7 +3358,7 @@ version = "0.5.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" dependencies = [ - "cfg_aliases", + "cfg_aliases 0.2.1", "libc", "once_cell", "socket2 0.6.1", @@ -4092,7 +4111,7 @@ dependencies = [ "env_logger 0.10.2", "hex", "msru", - "nix", + "nix 0.23.2", "openssl", "rand 0.8.5", "reqwest 0.11.10", diff --git a/Cargo.toml b/Cargo.toml index ba8956a..8cb0162 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -73,6 +73,7 @@ snpguest = { git = "https://github.com/faasm/snpguest.git" } subtle = "2.6.1" tempfile = "3.23.0" tokio = { version = "1" } +nix = "0.28" tokio-rustls = "0.26.2" uuid = { version = "^1.3" } walkdir = "2" diff --git a/attestation-service/bin/gen_keys.sh b/attestation-service/bin/gen_keys.sh deleted file mode 100755 index c6f3a77..0000000 --- a/attestation-service/bin/gen_keys.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]:-${(%):-%x}}" )" >/dev/null 2>&1 && pwd )" -PROJ_ROOT="${THIS_DIR}/.." - -URL="${AS_URL:-"127.0.0.1"}" -CERTS_DIR=${AS_CERTS_DIR:-"${PROJ_ROOT}/certs"} - -if [[ "$1" != "--force" ]]; then - echo "WARNING: this script overwrites the keys in ${CERTS_DIR}," - echo "WARNING: these keys and certificates are baked into the enclaves" - echo "WARNING: if you know what you are doing, re-run the script with --force" - exit 1 -fi - -mkdir -p ${CERTS_DIR} -openssl \ - req -x509 \ - -newkey rsa:4096 -keyout "${CERTS_DIR}/key.pem" \ - -out "${CERTS_DIR}/cert.pem" \ - -days 365 -nodes \ - -subj "/CN=${URL}" \ - -addext "subjectAltName = IP:${URL}" > /dev/null \ - && echo "accless: as: generated private key and certs at: ${CERTS_DIR}" \ - || echo "accless: as: error generating private key and certificates" diff --git a/attestation-service/src/main.rs b/attestation-service/src/main.rs index ab83cfd..52cb719 100644 --- a/attestation-service/src/main.rs +++ b/attestation-service/src/main.rs @@ -53,8 +53,14 @@ struct Cli { mock: bool, } -async fn health() -> impl IntoResponse { - (StatusCode::OK, "OK") +async fn health(Extension(state): Extension>) -> impl IntoResponse { + match tls::get_node_url() { + Ok(url) => (StatusCode::OK, format!("https://{}:{}", url, state.port)), + Err(err) => ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Error getting node URL: {}", err), + ), + } } #[tokio::main] @@ -71,9 +77,10 @@ async fn main() -> Result<()> { // Set-up per request state. let state = Arc::new(AttestationServiceState::new( - cli.certs_dir, + cli.certs_dir.clone(), cli.sgx_pccs_url.clone(), cli.mock, + cli.port, )?); // Start HTTPS server. @@ -86,8 +93,19 @@ async fn main() -> Result<()> { let addr = SocketAddr::from(([0, 0, 0, 0], cli.port)); let listener = TcpListener::bind(addr).await; - info!("Accless attestation server running on https://{}", addr); - info!("External IP: {}", tls::get_node_url()?); + info!("main(): accless attestation server running!"); + info!( + "main(): external IP: https://{}:{}", + tls::get_node_url()?, + cli.port + ); + info!( + "main(): cert path: {}/cert.pem", + cli.certs_dir + .unwrap_or(tls::get_default_certs_dir()) + .display() + ); + loop { let (stream, _) = listener.as_ref().expect("error listening").accept().await?; let service = TowerToHyperService::new(app.clone()); diff --git a/attestation-service/src/state.rs b/attestation-service/src/state.rs index c1daa9a..c5cdb5f 100644 --- a/attestation-service/src/state.rs +++ b/attestation-service/src/state.rs @@ -65,6 +65,8 @@ pub struct AttestationServiceState { #[cfg(feature = "azure-cvm")] pub vcek_pem: Vec, + /// Port the server is listening on. + pub port: u16, } impl AttestationServiceState { @@ -75,6 +77,7 @@ impl AttestationServiceState { certs_dir: Option, sgx_pccs_url: Option, mock_attestation: bool, + port: u16, ) -> Result { let certs_dir = certs_dir.unwrap_or_else(get_default_certs_dir); @@ -101,6 +104,7 @@ impl AttestationServiceState { amd_signing_keys: RwLock::new(BTreeMap::new()), #[cfg(feature = "snp")] snp_vcek_cache: RwLock::new(BTreeMap::new()), + port, }) } } diff --git a/attestation-service/src/tls.rs b/attestation-service/src/tls.rs index d7ee135..06bd73f 100644 --- a/attestation-service/src/tls.rs +++ b/attestation-service/src/tls.rs @@ -20,6 +20,7 @@ use tokio_rustls::TlsAcceptor; pub fn get_default_certs_dir() -> PathBuf { PathBuf::from(env!("ACCLESS_ROOT_DIR")) .join("config") + .join("attestation-service") .join("certs") } From da3043026e663ae7c6a08ad404ce8866b6985c8d Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Tue, 25 Nov 2025 17:01:01 +0000 Subject: [PATCH 25/52] [accli] E: Unify --as-* Flags --- accli/Cargo.toml | 1 + accli/src/main.rs | 16 ++++-- accli/src/tasks/applications.rs | 28 +++++++---- accli/src/tasks/attestation_service.rs | 69 ++++++++++++++++++++++---- accli/src/tasks/docker.rs | 10 ++-- 5 files changed, 96 insertions(+), 28 deletions(-) diff --git a/accli/Cargo.toml b/accli/Cargo.toml index e344676..a90669f 100644 --- a/accli/Cargo.toml +++ b/accli/Cargo.toml @@ -28,6 +28,7 @@ futures-util.workspace = true hex.workspace = true indicatif.workspace = true log.workspace = true +nix = { workspace = true, features = ["signal", "process"] } minio.workspace = true plotters.workspace = true rand.workspace = true diff --git a/accli/src/main.rs b/accli/src/main.rs index b3d2c71..940fc4f 100644 --- a/accli/src/main.rs +++ b/accli/src/main.rs @@ -94,7 +94,12 @@ enum AttestationServiceCommand { /// Rebuild the attestation service before running. #[arg(long, default_value_t = false)] rebuild: bool, + /// Run the attestation service in the background, storing its PID. + #[arg(long, default_value_t = false)] + background: bool, }, + /// Stop a running attestation service (started with --background). + Stop {}, Health { /// URL of the attestation service #[arg(long)] @@ -379,7 +384,7 @@ enum ApplicationsCommand { debug: bool, /// Path to the attestation service's public certificate PEM file. #[arg(long)] - cert_path: Option, + as_cert_path: Option, /// Whether to build the application inside a cVM. #[arg(long, default_value_t = false)] in_cvm: bool, @@ -425,10 +430,10 @@ async fn main() -> anyhow::Result<()> { ApplicationsCommand::Build { clean, debug, - cert_path, + as_cert_path, in_cvm, } => { - Applications::build(*clean, *debug, cert_path.as_deref(), false, *in_cvm)?; + Applications::build(*clean, *debug, as_cert_path.as_deref(), false, *in_cvm)?; } ApplicationsCommand::Run { app_type, @@ -896,6 +901,7 @@ async fn main() -> anyhow::Result<()> { force_clean_certs, mock, rebuild, + background, } => { AttestationService::run( certs_dir.as_deref(), @@ -904,8 +910,12 @@ async fn main() -> anyhow::Result<()> { *force_clean_certs, *mock, *rebuild, + *background, )?; } + AttestationServiceCommand::Stop {} => { + AttestationService::stop()?; + } AttestationServiceCommand::Health { url, cert_path } => { AttestationService::health(url.clone(), cert_path.clone()).await?; } diff --git a/accli/src/tasks/applications.rs b/accli/src/tasks/applications.rs index 4ffedca..fcb57b6 100644 --- a/accli/src/tasks/applications.rs +++ b/accli/src/tasks/applications.rs @@ -66,7 +66,7 @@ impl Applications { pub fn build( clean: bool, debug: bool, - cert_path: Option<&str>, + as_cert_path: Option<&str>, capture_output: bool, in_cvm: bool, ) -> Result> { @@ -92,8 +92,8 @@ impl Applications { if in_cvm { // Make sure the certificates are available in the cVM. let mut scp_files: Vec<(PathBuf, PathBuf)> = vec![]; - if let Some(cert_path_str) = cert_path { - let host_cert_path = PathBuf::from(cert_path_str); + if let Some(as_cert_path_str) = as_cert_path { + let host_cert_path = PathBuf::from(as_cert_path_str); if !host_cert_path.exists() { anyhow::bail!( "Certificate path does not exist: {}", @@ -113,7 +113,7 @@ impl Applications { ); scp_files.push((host_cert_path, guest_cert_path.clone())); - cmd.push("--cert-path".to_string()); + cmd.push("--as-cert-path".to_string()); cmd.push(guest_cert_path.display().to_string()); } @@ -128,8 +128,8 @@ impl Applications { )?; Ok(None) } else { - if let Some(cert_path_str) = cert_path { - let cert_path = Path::new(cert_path_str); + if let Some(as_cert_path_str) = as_cert_path { + let cert_path = Path::new(as_cert_path_str); if !cert_path.exists() { anyhow::bail!("Certificate path does not exist: {}", cert_path.display()); } @@ -137,7 +137,7 @@ impl Applications { anyhow::bail!("Certificate path is not a file: {}", cert_path.display()); } let docker_cert_path = Docker::get_docker_path(cert_path)?; - cmd.push("--cert-path".to_string()); + cmd.push("--as-cert-path".to_string()); let docker_cert_path_str = docker_cert_path.to_str().ok_or_else(|| { anyhow::anyhow!( "Docker path for certificate is not valid UTF-8: {}", @@ -164,7 +164,7 @@ impl Applications { // If --in-cvm flag is passed, we literally re run the same `accli` command, but // inside the cVM. if in_cvm { - let cmd = vec![ + let mut cmd = vec![ "./scripts/accli_wrapper.sh".to_string(), "applications".to_string(), "run".to_string(), @@ -172,6 +172,16 @@ impl Applications { format!("{app_name}"), ]; + if let Some(as_url) = as_url { + cmd.push("--as-url".to_string()); + cmd.push(as_url.to_string()); + } + + if let Some(as_cert_path) = as_cert_path { + cmd.push("--as-cert-path".to_string()); + cmd.push(format!("{}", as_cert_path.display())); + } + // We don't need to SCP any files here, because we assume that the certificates // have been copied during the build stage, and persisted in the // disk image. @@ -187,7 +197,7 @@ impl Applications { // Note that we need to make the plural. .join(format!("{app_type}s")) .join(format!("{app_name}")) - .join(format!("{app_name}")) + .join(format!("{app_name}")), }; let binary_path_str = binary_path.to_str().ok_or_else(|| { diff --git a/accli/src/tasks/attestation_service.rs b/accli/src/tasks/attestation_service.rs index 6173436..9156be3 100644 --- a/accli/src/tasks/attestation_service.rs +++ b/accli/src/tasks/attestation_service.rs @@ -1,6 +1,10 @@ use crate::env::Env; -use anyhow::Result; +use anyhow::{Context, Result}; use log::info; +use nix::{ + sys::signal::{Signal, kill}, + unistd::Pid, +}; use reqwest; use std::{ fs, @@ -9,6 +13,7 @@ use std::{ const AS_URL_ENV_VAR: &str = "ACCLESS_AS_URL"; const AS_CERT_PATH_ENV_VAR: &str = "ACCLESS_AS_CERT_PATH"; +const PID_FILE_PATH: &str = "./config/attestation-service/PID"; /// Returns the required attestation-service env. vars given a URL and cert /// path. @@ -42,6 +47,7 @@ impl AttestationService { force_clean_certs: bool, mock: bool, rebuild: bool, + background: bool, ) -> Result<()> { if rebuild { Self::build()?; @@ -68,13 +74,46 @@ impl AttestationService { if mock { cmd.arg("--mock"); } - let status = cmd - .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()) - .status()?; - if !status.success() { - anyhow::bail!("Failed to run attestation service"); + + if background { + info!("run(): running attestation service in background..."); + let child = cmd + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .context("Failed to spawn attestation service in background")?; + fs::write(PID_FILE_PATH, child.id().to_string()).context("Failed to write PID file")?; + info!("run(): attestation service spawned (PID={})", child.id()); + Ok(()) + } else { + let status = cmd + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .status()?; + if !status.success() { + anyhow::bail!("Failed to run attestation service"); + } + Ok(()) } + } + + pub fn stop() -> Result<()> { + info!("stop(): stopping attestation service..."); + let pid_str = fs::read_to_string(PID_FILE_PATH).context(format!( + "Failed to read PID file at {}. Is the service running in background?", + PID_FILE_PATH + ))?; + let pid: u32 = pid_str + .trim() + .parse() + .context(format!("Failed to parse PID from file {}", PID_FILE_PATH))?; + + info!("stop(): killing process with PID: {}", pid); + kill(Pid::from_raw(pid as i32), Signal::SIGTERM) + .context(format!("Failed to kill process with PID {}", pid))?; + + fs::remove_file(PID_FILE_PATH).context("failed to remove PID file")?; + info!("stop(): ttestation service stopped (PID={})", pid); Ok(()) } @@ -105,8 +144,18 @@ impl AttestationService { }?; let response = client.get(format!("{}/health", url)).send().await?; - info!("Health check response: {}", response.text().await?); - - Ok(()) + if response.status().is_success() { + let body = response.text().await?; + info!( + "health(): attestation service is healthy and reachable on: {}", + body + ); + Ok(()) + } else { + anyhow::bail!( + "health(): attestation service is not healthy (status={})", + response.status() + ); + } } } diff --git a/accli/src/tasks/docker.rs b/accli/src/tasks/docker.rs index 2ab789a..40999c9 100644 --- a/accli/src/tasks/docker.rs +++ b/accli/src/tasks/docker.rs @@ -195,9 +195,9 @@ impl Docker { /// Helper method to get the group ID of the /dev/sev-guest device. /// - /// This method is only used when using `accli` inside a cVM. In our cVM set-up we configure - /// /dev/sev-guest to be in a shared group with our user, to avoid having to use `sudo` to run - /// our functions. + /// This method is only used when using `accli` inside a cVM. In our cVM + /// set-up we configure /dev/sev-guest to be in a shared group with our + /// user, to avoid having to use `sudo` to run our functions. fn get_sevguest_group_id() -> Option { match std::fs::metadata("/dev/sev-guest") { Ok(metadata) => Some(metadata.gid()), @@ -274,9 +274,7 @@ impl Docker { .arg(format!("HOST_GID={}", Self::get_group_id())); if let Some(sevgest_gid) = Self::get_sevguest_group_id() { - run_cmd - .arg("-e") - .arg(format!("SEV_GID={}", sevgest_gid)); + run_cmd.arg("-e").arg(format!("SEV_GID={}", sevgest_gid)); } for e in env { From 1da30df24ed27345700acd8269f9d1d30878b2ba Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Tue, 25 Nov 2025 17:01:25 +0000 Subject: [PATCH 26/52] [applications] B: Adapt --as-cert-path Flag --- applications/build.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/applications/build.py b/applications/build.py index 0743d91..5ed306c 100644 --- a/applications/build.py +++ b/applications/build.py @@ -10,7 +10,7 @@ PROJ_ROOT = dirname(APPS_ROOT) -def compile(wasm=False, native=False, debug=False, clean=False, cert_path=None): +def compile(wasm=False, native=False, debug=False, clean=False, as_cert_path=None): """ Compile the different applications supported in Accless. """ @@ -24,11 +24,11 @@ def compile(wasm=False, native=False, debug=False, clean=False, cert_path=None): if not exists(build_dir): makedirs(build_dir) - if cert_path is not None: - if not exists(cert_path): + if as_cert_path is not None: + if not exists(as_cert_path): print(f"ERROR: passed --cert-path variable but path does not exist") exit(1) - cert_path = abspath(cert_path) + as_cert_path = abspath(as_cert_path) # if wasm: # wasm_cmake( @@ -47,7 +47,7 @@ def compile(wasm=False, native=False, debug=False, clean=False, cert_path=None): "-DCMAKE_BUILD_TYPE={}".format("Debug" if debug else "Release"), "-DCMAKE_C_COMPILER=/usr/bin/clang-17", "-DCMAKE_CXX_COMPILER=/usr/bin/clang++-17", - f"-DACCLESS_AS_CERT_PEM={cert_path}" if cert_path is not None else "", + f"-DACCLESS_AS_CERT_PEM={as_cert_path}" if as_cert_path is not None else "", APPS_ROOT, ] cmake_cmd = " ".join(cmake_cmd) @@ -84,11 +84,11 @@ def compile(wasm=False, native=False, debug=False, clean=False, cert_path=None): wasm=True, debug=args.debug, clean=args.clean, - cert_path=args.cert_path, + as_cert_path=args.as_cert_path, ) compile( native=True, debug=args.debug, clean=args.clean, - cert_path=args.cert_path, + as_cert_path=args.as_cert_path, ) From 71ad62617c1841e2ebe98c54861233dc9eafd69a Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Tue, 25 Nov 2025 17:01:59 +0000 Subject: [PATCH 27/52] [scripts] E: Build Accli In Snp Setup --- scripts/snp/setup.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/snp/setup.sh b/scripts/snp/setup.sh index 395f05c..cf0ae4a 100755 --- a/scripts/snp/setup.sh +++ b/scripts/snp/setup.sh @@ -231,7 +231,7 @@ systemctl enable ssh > /dev/null 2>&1 || systemctl enable ssh.service > /dev/nul echo "[provision/chroot] Installing rustup for ubuntu..." su -l ubuntu -c 'curl --proto "=https" --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y > /dev/null 2>&1' || true -# FIXME: remove branch name and build some deps. +# FIXME: remove branch name. echo "[provision/chroot] Cloning Accless repo (idempotent)..." su -l ubuntu -c ' cd /home/ubuntu && @@ -240,6 +240,7 @@ su -l ubuntu -c ' else echo "accless repo already present, skipping clone"; fi + cd /home/ubuntu/accless && cargo build -p accli --release ' || true # (Optional) You *could* pull docker images here, but it's messy because dockerd From 4d02d779599502f83feca451cde633f2c3aced5d Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Tue, 25 Nov 2025 17:02:16 +0000 Subject: [PATCH 28/52] [ci] E: Add SNP Bare Metal Test --- .github/workflows/snp.yml | 52 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 .github/workflows/snp.yml diff --git a/.github/workflows/snp.yml b/.github/workflows/snp.yml new file mode 100644 index 0000000..31da000 --- /dev/null +++ b/.github/workflows/snp.yml @@ -0,0 +1,52 @@ +name: "SNP End-to-End Tests" + +on: + push: + branches: [main] + pull_request: + branches: [main] + types: [opened, synchronize, reopened, ready_for_review] + + +defaults: + run: + shell: bash + +# Cancel previous running actions for the same PR +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +jobs: + run-functions: + runs-on: [self-hosted, snp] + steps: + - name: "Check out the code" + uses: actions/checkout@v4 + + - name: "Cache SNP artefacts" + id: cache-snp + uses: actions/cache@v4 + with: + path: scripts/snp/output + key: snp-setup-${{ runner.os }}-${{ hashFiles('scripts/snp/**') }} + + # Only re-run on a cache miss (setup may take a while) + - name: "Run SNP setup" + if: steps.cache-snp.outputs.cache-hit != 'true' + run: ./scripts/accli_wrapper.sh dev cvm setup --clean + + # This step may take a while. + - name: "Start attestation service in the background" + run: ./scripts/accli_wrapper.sh attestation-service run --background --certs-dir ./certs --force-clean-certs + + # Build SNP applications and embed the attestation service's certificate. + - name: "Build SNP applications" + run: ./scrips/accli_wrapper.sh applications build --clean --as-cert-path ./certs/cert.pem --in-cvm + + - name: "Run supported SNP applications" + run: | + # First get the external IP so that we can reach the attestation-service from the cVM. + AS_URL=$(accli attestation-service health --url "https://0.0.0.0:8443" --cert-path ./certs/cert.pem \ + | grep "attestation service is healthy and reachable on:" | awk '{print $NF}') + ./scripts/accli_wrapper.sh applications run function escrow-xput --as-url ${AS_URL} --as-cert-path ./certs/cert.pem From 43a697bc503607ca4f1546a711c3e9549d3b1a01 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Tue, 25 Nov 2025 17:04:04 +0000 Subject: [PATCH 29/52] [build] E: Fix num-bigint Warning --- Cargo.lock | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ba41eff..4e03522 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2810,11 +2810,10 @@ dependencies = [ [[package]] name = "num-bigint-dig" -version = "0.8.4" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" dependencies = [ - "byteorder", "lazy_static", "libm", "num-integer", From b823653618ddcfa4d7dc7492d6974031cc404994 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Tue, 25 Nov 2025 17:08:58 +0000 Subject: [PATCH 30/52] [attestation-service] W: Clarify azcVM Regression --- .github/workflows/snp.yml | 2 +- attestation-service/src/azure_cvm.rs | 24 ++++-------------------- 2 files changed, 5 insertions(+), 21 deletions(-) diff --git a/.github/workflows/snp.yml b/.github/workflows/snp.yml index 31da000..78d5268 100644 --- a/.github/workflows/snp.yml +++ b/.github/workflows/snp.yml @@ -42,7 +42,7 @@ jobs: # Build SNP applications and embed the attestation service's certificate. - name: "Build SNP applications" - run: ./scrips/accli_wrapper.sh applications build --clean --as-cert-path ./certs/cert.pem --in-cvm + run: ./scripts/accli_wrapper.sh applications build --clean --as-cert-path ./certs/cert.pem --in-cvm - name: "Run supported SNP applications" run: | diff --git a/attestation-service/src/azure_cvm.rs b/attestation-service/src/azure_cvm.rs index c10a396..1ed6b83 100644 --- a/attestation-service/src/azure_cvm.rs +++ b/attestation-service/src/azure_cvm.rs @@ -9,25 +9,9 @@ struct VcekResponse { } /// This method can only be called from an Azure cVM -/// FIXME: gate be pub fn fetch_vcek_pem() -> Result, Box> { - match ureq::get("http://169.254.169.254/metadata/THIM/amd/certification") - .set("Metadata", "true") - .call() - { - Ok(resp) => match resp.into_json::() { - Ok(data) => { - let pem = format!("{}\n{}", data.vcek_cert, data.certificate_chain); - Ok(pem.into_bytes()) - } - Err(e) => { - warn!("failed to parse VCECK response JSON: {e}"); - Ok(vec![]) - } - }, - Err(e) => { - warn!("failed to fetch VCECK certificates: {e}"); - Ok(vec![]) - } - } + // FIXME(#55): re-introduce azure cvm quote validation. In the past we used this URL to fetch + // VCEK certificates: + // http://169.254.169.254/metadata/THIM/amd/certification + Ok(vec![]) } From dfdca48f4fa97f889391962d0628f1d7b8ffe9d5 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Tue, 25 Nov 2025 17:34:46 +0000 Subject: [PATCH 31/52] [attestation-service] B: Fix Default Cert Path --- attestation-service/src/tls.rs | 2 +- attestation-service/tests/main.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/attestation-service/src/tls.rs b/attestation-service/src/tls.rs index 06bd73f..eaaeb61 100644 --- a/attestation-service/src/tls.rs +++ b/attestation-service/src/tls.rs @@ -221,7 +221,7 @@ mod tests { #[test] fn test_get_certs_dir() { let certs_dir = get_default_certs_dir(); - assert!(certs_dir.ends_with("config/certs")); + assert!(certs_dir.ends_with("config/attestation-service/certs")); } #[test] diff --git a/attestation-service/tests/main.rs b/attestation-service/tests/main.rs index a5bf9d0..5ba02dd 100644 --- a/attestation-service/tests/main.rs +++ b/attestation-service/tests/main.rs @@ -128,7 +128,7 @@ async fn test_spawn_as_no_clean() -> Result<()> { #[tokio::test] #[serial] async fn test_att_clients() -> Result<()> { - attestation_service::init_logging(true); + attestation_service::init_logging(); let certs_dir = Path::new(env!("ACCLESS_ROOT_DIR")) .join("config") @@ -148,7 +148,7 @@ async fn test_att_clients() -> Result<()> { // container. We also _must_ set the `clean` flag to true, to force // recompilation. info!("re-building mock clients with new certificates, this will take a while..."); - Applications::build(true, false, cert_path.to_str(), true)?; + Applications::build(true, false, cert_path.to_str(), true, false)?; // Health-check the attestation service. let client = reqwest::Client::builder() From e2d4da97f40f176210db1c1f3bac7b37f0b853f8 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Tue, 25 Nov 2025 17:35:07 +0000 Subject: [PATCH 32/52] [accli] B: Fix cVM Usage Script --- accli/src/tasks/cvm.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/accli/src/tasks/cvm.rs b/accli/src/tasks/cvm.rs index 1209da8..17f6769 100644 --- a/accli/src/tasks/cvm.rs +++ b/accli/src/tasks/cvm.rs @@ -187,12 +187,13 @@ pub fn build(clean: bool, component: Option) -> Result<()> { /// /// # Example Usage /// -/// ``` +/// ```rust,no_run +/// use accli::tasks::cvm; /// use anyhow::Result; /// use std::path::PathBuf; /// /// // Example of running a command without SCPing files -/// cvm::run(&["ls".to_string(), "-la".to_string()], None).unwrap(); +/// cvm::run(&["ls".to_string(), "-la".to_string()], None, None).unwrap(); /// /// // Example of SCPing a file and then running a command /// let host_path = PathBuf::from("./my_local_file.txt"); @@ -200,6 +201,7 @@ pub fn build(clean: bool, component: Option) -> Result<()> { /// cvm::run( /// &["cat".to_string(), "my_remote_file.txt".to_string()], /// Some(&[(host_path, guest_path)]), +/// None, /// ) /// .unwrap(); /// ``` From bf40874448f467243c3cbf7d1c52bc18ce6d8ab7 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Tue, 25 Nov 2025 17:35:35 +0000 Subject: [PATCH 33/52] [scripts] W: Avoid edksetup.sh Crashing --- scripts/snp/setup.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/snp/setup.sh b/scripts/snp/setup.sh index cf0ae4a..b85d851 100755 --- a/scripts/snp/setup.sh +++ b/scripts/snp/setup.sh @@ -63,7 +63,10 @@ build_ovmf() { git submodule update --init --recursive > /dev/null 2>&1 make -C BaseTools clean > /dev/null 2>&1 make -C BaseTools -j $(nproc) > /dev/null 2>&1 - . ./edksetup.sh --reconfig + # edksetup supports having unbound variables, so we must temporarily enable them. + set +u + . ./edksetup.sh --reconfig > /dev/null 2>&1 + set -u build -a X64 -b RELEASE -t GCC5 -p OvmfPkg/OvmfPkgX64.dsc > /dev/null 2>&1 touch OvmfPkg/AmdSev/Grub/grub.efi > /dev/null 2>&1 build -a X64 -b RELEASE -t GCC5 -p OvmfPkg/AmdSev/AmdSevX64.dsc > /dev/null 2>&1 From 47c63ba7bc55e14a452f6b2920d058b9340800e6 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Tue, 25 Nov 2025 17:36:02 +0000 Subject: [PATCH 34/52] [ci] B: Only Run SNP Job On Non-Draft PRs --- .github/workflows/snp.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/snp.yml b/.github/workflows/snp.yml index 78d5268..dd68fb5 100644 --- a/.github/workflows/snp.yml +++ b/.github/workflows/snp.yml @@ -19,6 +19,7 @@ concurrency: jobs: run-functions: + if: github.event.pull_request.draft == false runs-on: [self-hosted, snp] steps: - name: "Check out the code" From 79dedcca1290c1084f2d7c7e62e67775c17631e5 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Tue, 25 Nov 2025 17:48:34 +0000 Subject: [PATCH 35/52] [scripts] E: Refine SNP Setup Script --- scripts/snp/setup.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/snp/setup.sh b/scripts/snp/setup.sh index b85d851..d222b68 100755 --- a/scripts/snp/setup.sh +++ b/scripts/snp/setup.sh @@ -67,9 +67,9 @@ build_ovmf() { set +u . ./edksetup.sh --reconfig > /dev/null 2>&1 set -u - build -a X64 -b RELEASE -t GCC5 -p OvmfPkg/OvmfPkgX64.dsc > /dev/null 2>&1 + build -n $(nproc) -a X64 -b RELEASE -t GCC5 -p OvmfPkg/OvmfPkgX64.dsc > /dev/null 2>&1 touch OvmfPkg/AmdSev/Grub/grub.efi > /dev/null 2>&1 - build -a X64 -b RELEASE -t GCC5 -p OvmfPkg/AmdSev/AmdSevX64.dsc > /dev/null 2>&1 + build -n $(nproc) -a X64 -b RELEASE -t GCC5 -p OvmfPkg/AmdSev/AmdSevX64.dsc > /dev/null 2>&1 popd >> /dev/null popd >> /dev/null @@ -137,8 +137,8 @@ provision_disk_image() { # Grow the ext4 filesystem to occupy all the disk space. print_info "[provision-disk] Growing filesystem..." sudo parted /dev/nbd0 --script resizepart 1 100% - sudo e2fsck -f ${root_dev} # > /dev/null 2>&1 - sudo resize2fs ${root_dev} # > /dev/null 2>&1 + sudo e2fsck -fy ${root_dev} + sudo resize2fs ${root_dev} print_info "[provision-disk] Mounting root filesystem ${root_dev} at ${root_mnt}..." sudo mkdir -p "${root_mnt}" From 7e915c563451cb18361722b696a15db2ea6a23e0 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Tue, 25 Nov 2025 18:13:24 +0000 Subject: [PATCH 36/52] [ci] B: Install APT Deps --- .github/workflows/tests.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8124e9f..fe14a32 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -36,6 +36,8 @@ jobs: steps: - name: "Checkout code" uses: actions/checkout@v4 + - name: "Install APT deps" + run: ./scripts/apt.sh - name: "Run Rust unit tests" run: | source ./scripts/workon.sh @@ -52,9 +54,9 @@ jobs: steps: - name: "Checkout code" uses: actions/checkout@v4 + - name: "Install APT deps" + run: ./scripts/apt.sh - name: "Build C++ code" - shell: bash run: ./scripts/accli_wrapper.sh accless build --clean - name: "Run C++ unit tests" - shell: bash run: ./scripts/accli_wrapper.sh accless test From 5128f13eccc90093b127ba19829253b6fde910a9 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Tue, 25 Nov 2025 18:59:01 +0000 Subject: [PATCH 37/52] [ci] B: Build AS In SNP Tests --- .github/workflows/snp.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/snp.yml b/.github/workflows/snp.yml index dd68fb5..a651910 100644 --- a/.github/workflows/snp.yml +++ b/.github/workflows/snp.yml @@ -39,7 +39,7 @@ jobs: # This step may take a while. - name: "Start attestation service in the background" - run: ./scripts/accli_wrapper.sh attestation-service run --background --certs-dir ./certs --force-clean-certs + run: ./scripts/accli_wrapper.sh attestation-service run --background --certs-dir ./certs --force-clean-certs --rebuild # Build SNP applications and embed the attestation service's certificate. - name: "Build SNP applications" From 7a144a5327fa37143e146af16758ad5f7959a215 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Wed, 26 Nov 2025 11:44:45 +0000 Subject: [PATCH 38/52] [attestation-service] E: Refactor API Tests --- attestation-service/src/azure_cvm.rs | 4 +- .../tests/{main.rs => as_api_tests.rs} | 122 +++++++----------- attestation-service/tests/common.rs | 28 ---- 3 files changed, 52 insertions(+), 102 deletions(-) rename attestation-service/tests/{main.rs => as_api_tests.rs} (61%) delete mode 100644 attestation-service/tests/common.rs diff --git a/attestation-service/src/azure_cvm.rs b/attestation-service/src/azure_cvm.rs index 1ed6b83..9aa51cd 100644 --- a/attestation-service/src/azure_cvm.rs +++ b/attestation-service/src/azure_cvm.rs @@ -10,8 +10,8 @@ struct VcekResponse { /// This method can only be called from an Azure cVM pub fn fetch_vcek_pem() -> Result, Box> { - // FIXME(#55): re-introduce azure cvm quote validation. In the past we used this URL to fetch - // VCEK certificates: + // FIXME(#55): re-introduce azure cvm quote validation. In the past we used this + // URL to fetch VCEK certificates: // http://169.254.169.254/metadata/THIM/amd/certification Ok(vec![]) } diff --git a/attestation-service/tests/main.rs b/attestation-service/tests/as_api_tests.rs similarity index 61% rename from attestation-service/tests/main.rs rename to attestation-service/tests/as_api_tests.rs index 5ba02dd..89e45a2 100644 --- a/attestation-service/tests/main.rs +++ b/attestation-service/tests/as_api_tests.rs @@ -1,7 +1,4 @@ -use accli::tasks::{ - applications::Applications, - docker::{DOCKER_ACCLESS_CODE_MOUNT_DIR, Docker}, -}; +use accli::tasks::applications::{ApplicationName, ApplicationType, Applications}; use anyhow::Result; use log::{error, info}; use reqwest::Client; @@ -9,22 +6,46 @@ use serde_json::Value; use serial_test::serial; use std::{ path::{Path, PathBuf}, + process::Stdio, time::Duration, }; use tempfile::tempdir; -use tokio::process::Child; - -mod common; +use tokio::process::{Child, Command}; struct ChildGuard(Child); +// =============================================================================================== +// Helper Functions +// =============================================================================================== + impl Drop for ChildGuard { fn drop(&mut self) { let _ = self.0.start_kill(); } } -// FIXME: I doubt this is working +fn spawn_as(certs_dir: &str, clean_certs: bool, mock: bool) -> Result { + let mut cmd = Command::new(env!("CARGO_BIN_EXE_attestation-service")); + cmd.arg("--certs-dir") + .arg(certs_dir) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()); + + if clean_certs { + cmd.arg("--force-clean-certs"); + } + + if mock { + cmd.arg("--mock"); + } + + Ok(cmd.spawn()?) +} + +pub fn get_public_certificate_path(certs_dir: &Path) -> PathBuf { + certs_dir.join("cert.pem") +} + async fn health_check(client: &Client) -> Result<()> { let mut attempts = 0; let max_attempts = 5; @@ -48,49 +69,16 @@ async fn health_check(client: &Client) -> Result<()> { } } -/// # Description -/// -/// Get the path of SGX's attestation client from _inside_ the docker container. -fn get_att_client_sgx_path_in_ctr() -> Result { - let path = PathBuf::from(DOCKER_ACCLESS_CODE_MOUNT_DIR) - .join("applications") - .join("build-native") - .join("test") - .join("att-client-sgx") - .join("att-client-sgx"); - - Ok(path) -} - -/// # Description -/// -/// Get the path of SNP's attestation client from _inside_ the docker container. -fn get_att_client_snp_path_in_ctr() -> Result { - let path = PathBuf::from(DOCKER_ACCLESS_CODE_MOUNT_DIR) - .join("applications") - .join("build-native") - .join("test") - .join("att-client-snp") - .join("att-client-snp"); - - Ok(path) -} - -/// # Description -/// -/// Remap an absolute path the host to the mounted container. -pub fn remap_host_path_to_container(host_path: &Path) -> Result { - let prefix = Path::new(env!("ACCLESS_ROOT_DIR")); - let rel = host_path.strip_prefix(prefix)?; - Ok(Path::new("/code/accless").join(rel)) -} +// =============================================================================================== +// Tests +// =============================================================================================== #[tokio::test] #[serial] async fn test_spawn_as() -> Result<()> { let temp_dir = tempdir()?; let certs_dir = temp_dir.path(); - let child = common::spawn_as(certs_dir.to_str().unwrap(), true, false)?; + let child = spawn_as(certs_dir.to_str().unwrap(), true, false)?; let _child_guard = ChildGuard(child); // Give the service time to start. @@ -109,7 +97,7 @@ async fn test_spawn_as() -> Result<()> { async fn test_spawn_as_no_clean() -> Result<()> { let temp_dir = tempdir()?; let certs_dir = temp_dir.path(); - let child = common::spawn_as(certs_dir.to_str().unwrap(), false, false)?; + let child = spawn_as(certs_dir.to_str().unwrap(), false, false)?; let _child_guard = ChildGuard(child); // Give the service time to start. tokio::time::sleep(Duration::from_secs(2)).await; @@ -122,9 +110,6 @@ async fn test_spawn_as_no_clean() -> Result<()> { Ok(()) } -/// WARNING: this test relies on `accli` and on the test applications being -/// compiled. The former can be (re-)compiled with `cargo build -p accli` and -/// the latter with `accli applications build test` #[tokio::test] #[serial] async fn test_att_clients() -> Result<()> { @@ -132,15 +117,15 @@ async fn test_att_clients() -> Result<()> { let certs_dir = Path::new(env!("ACCLESS_ROOT_DIR")) .join("config") + .join("attestation-service") .join("test-certs"); - let child = common::spawn_as(certs_dir.to_str().unwrap(), true, true)?; + let child = spawn_as(certs_dir.to_str().unwrap(), true, true)?; let _child_guard = ChildGuard(child); // Give the service time to start. tokio::time::sleep(Duration::from_secs(2)).await; - let cert_path = - remap_host_path_to_container(&crate::common::get_public_certificate_path(&certs_dir))?; + let cert_path = get_public_certificate_path(&certs_dir); // While it is starting, rebuild the test application so that we can inject the // new certificates. Note that we need to pass the certificate's path @@ -148,7 +133,7 @@ async fn test_att_clients() -> Result<()> { // container. We also _must_ set the `clean` flag to true, to force // recompilation. info!("re-building mock clients with new certificates, this will take a while..."); - Applications::build(true, false, cert_path.to_str(), true, false)?; + Applications::build(true, false, Some(cert_path.clone()), true, false)?; // Health-check the attestation service. let client = reqwest::Client::builder() @@ -157,31 +142,24 @@ async fn test_att_clients() -> Result<()> { health_check(&client).await?; // Run the client applications inside the container. - let att_client_sgx_path = get_att_client_sgx_path_in_ctr()?; - let att_client_snp_path = get_att_client_snp_path_in_ctr()?; - let env_vars = [ - "ACCLESS_AS_URL=https://127.0.0.1:8443".to_string(), - format!("ACCLESS_AS_CERT_PATH={}", cert_path.display()), - ]; + let as_url = "https://127.0.0.1:8443".to_string(); info!("running mock sgx client..."); - Docker::run( - &[att_client_sgx_path.display().to_string()], - true, - None, - &env_vars, - true, + Applications::run( + ApplicationType::Test, + ApplicationName::AttClientSgx, false, + Some(as_url.clone()), + Some(cert_path.clone()), )?; info!("running mock snp client..."); - Docker::run( - &[att_client_snp_path.display().to_string()], - true, - None, - &env_vars, - true, + Applications::run( + ApplicationType::Test, + ApplicationName::AttClientSnp, false, + Some(as_url), + Some(cert_path), )?; match std::fs::remove_dir_all(&certs_dir) { @@ -200,7 +178,7 @@ async fn test_att_clients() -> Result<()> { async fn test_get_state() -> Result<()> { let temp_dir = tempdir()?; let certs_dir = temp_dir.path(); - let child = common::spawn_as(certs_dir.to_str().unwrap(), true, false)?; + let child = spawn_as(certs_dir.to_str().unwrap(), true, false)?; let _child_guard = ChildGuard(child); let client = reqwest::Client::builder() diff --git a/attestation-service/tests/common.rs b/attestation-service/tests/common.rs deleted file mode 100644 index cfeaa81..0000000 --- a/attestation-service/tests/common.rs +++ /dev/null @@ -1,28 +0,0 @@ -use anyhow::Result; -use std::{ - path::{Path, PathBuf}, - process::Stdio, -}; -use tokio::process::{Child, Command}; - -pub fn spawn_as(certs_dir: &str, clean_certs: bool, mock: bool) -> Result { - let mut cmd = Command::new(env!("CARGO_BIN_EXE_attestation-service")); - cmd.arg("--certs-dir") - .arg(certs_dir) - .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()); - - if clean_certs { - cmd.arg("--force-clean-certs"); - } - - if mock { - cmd.arg("--mock"); - } - - Ok(cmd.spawn()?) -} - -pub fn get_public_certificate_path(certs_dir: &Path) -> PathBuf { - certs_dir.join("cert.pem") -} From f550f0cb6d2a1b90970e108e5d00ae53f4a698b0 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Wed, 26 Nov 2025 11:45:32 +0000 Subject: [PATCH 39/52] [accli] E: Homogeneize AS Cert Path Parsing --- accli/src/main.rs | 8 +- accli/src/tasks/applications.rs | 150 ++++++++++++++++++-------------- accli/src/tasks/cvm.rs | 38 +++++++- accli/src/tasks/docker.rs | 3 +- 4 files changed, 130 insertions(+), 69 deletions(-) diff --git a/accli/src/main.rs b/accli/src/main.rs index 940fc4f..3b75b19 100644 --- a/accli/src/main.rs +++ b/accli/src/main.rs @@ -190,7 +190,7 @@ enum DockerCommand { /// Build one of Accless' docker containers. Run build --help to see the /// possibe options Build { - #[arg(short, long, num_args = 1.., value_name = "CTR_NAME")] + /// Container image to build. ctr: Vec, #[arg(long)] push: bool, @@ -384,7 +384,7 @@ enum ApplicationsCommand { debug: bool, /// Path to the attestation service's public certificate PEM file. #[arg(long)] - as_cert_path: Option, + as_cert_path: Option, /// Whether to build the application inside a cVM. #[arg(long, default_value_t = false)] in_cvm: bool, @@ -394,7 +394,7 @@ enum ApplicationsCommand { /// Type of the application to run app_type: applications::ApplicationType, /// Name of the application to run - app_name: applications::Functions, + app_name: applications::ApplicationName, /// Whether to run the application inside a cVM. #[arg(long, default_value_t = false)] in_cvm: bool, @@ -433,7 +433,7 @@ async fn main() -> anyhow::Result<()> { as_cert_path, in_cvm, } => { - Applications::build(*clean, *debug, as_cert_path.as_deref(), false, *in_cvm)?; + Applications::build(*clean, *debug, as_cert_path.clone(), false, *in_cvm)?; } ApplicationsCommand::Run { app_type, diff --git a/accli/src/tasks/applications.rs b/accli/src/tasks/applications.rs index fcb57b6..e7ced13 100644 --- a/accli/src/tasks/applications.rs +++ b/accli/src/tasks/applications.rs @@ -2,8 +2,9 @@ use crate::tasks::{ attestation_service, cvm, docker::{DOCKER_ACCLESS_CODE_MOUNT_DIR, Docker}, }; -use anyhow::{Context, Result}; +use anyhow::Result; use clap::ValueEnum; +use log::error; use std::{ fmt::{Display, Formatter, Result as FmtResult}, path::{Path, PathBuf}, @@ -13,12 +14,14 @@ use std::{ #[derive(Clone, Debug, ValueEnum)] pub enum ApplicationType { Function, + Test, } impl Display for ApplicationType { fn fmt(&self, f: &mut Formatter) -> FmtResult { match self { ApplicationType::Function => write!(f, "function"), + ApplicationType::Test => write!(f, "test"), } } } @@ -35,30 +38,76 @@ impl FromStr for ApplicationType { } #[derive(Clone, Debug, ValueEnum)] -pub enum Functions { +pub enum ApplicationName { + #[value(name = "att-client-sgx")] + AttClientSgx, + #[value(name = "att-client-snp")] + AttClientSnp, #[value(name = "escrow-xput")] EscrowXput, } -impl Display for Functions { +impl Display for ApplicationName { fn fmt(&self, f: &mut Formatter) -> FmtResult { match self { - Functions::EscrowXput => write!(f, "escrow-xput"), + ApplicationName::AttClientSgx => write!(f, "att-client-sgx"), + ApplicationName::AttClientSnp => write!(f, "att-client-snp"), + ApplicationName::EscrowXput => write!(f, "escrow-xput"), } } } -impl FromStr for Functions { +impl FromStr for ApplicationName { type Err = anyhow::Error; fn from_str(s: &str) -> Result { match s { - "escrow-xput" => Ok(Functions::EscrowXput), + "att-client-sgx" => Ok(ApplicationName::AttClientSgx), + "att-client-snp" => Ok(ApplicationName::AttClientSnp), + "escrow-xput" => Ok(ApplicationName::EscrowXput), _ => anyhow::bail!("Invalid Function: {}", s), } } } +fn host_cert_path_to_target_path( + as_cert_path: &Path, + in_cvm: bool, + in_docker: bool, +) -> Result { + if in_cvm & in_docker { + let reason = "cannot set in_cvm and in_docker"; + error!("as_cert_path_arg_to_real_path(): {reason}"); + anyhow::bail!(reason); + } + + if !as_cert_path.exists() { + let reason = format!( + "as certificate path does not exist (path={})", + as_cert_path.display() + ); + error!("as_cert_path_arg_to_real_path(): {reason}"); + anyhow::bail!(reason); + } + + if !as_cert_path.is_file() { + let reason = format!( + "as certificate path does not point to a file (path={})", + as_cert_path.display() + ); + error!("as_cert_path_arg_to_real_path(): {reason}"); + anyhow::bail!(reason); + } + + if in_docker { + Ok(Docker::remap_to_docker_path(as_cert_path)?) + } else if in_cvm { + Ok(cvm::remap_to_cvm_path(as_cert_path)?) + } else { + Ok(as_cert_path.to_path_buf()) + } +} + #[derive(Debug)] pub struct Applications {} @@ -66,7 +115,7 @@ impl Applications { pub fn build( clean: bool, debug: bool, - as_cert_path: Option<&str>, + as_cert_path: Option, capture_output: bool, in_cvm: bool, ) -> Result> { @@ -89,28 +138,13 @@ impl Applications { cmd.push("--debug".to_string()); } + // Work-out the right cert path depending on whether we are gonna SSH into the + // cVM or not. if in_cvm { // Make sure the certificates are available in the cVM. let mut scp_files: Vec<(PathBuf, PathBuf)> = vec![]; - if let Some(as_cert_path_str) = as_cert_path { - let host_cert_path = PathBuf::from(as_cert_path_str); - if !host_cert_path.exists() { - anyhow::bail!( - "Certificate path does not exist: {}", - host_cert_path.display() - ); - } - if !host_cert_path.is_file() { - anyhow::bail!( - "Certificate path is not a file: {}", - host_cert_path.display() - ); - } - let guest_cert_path = PathBuf::from("applications").join( - host_cert_path - .file_name() - .context("Failed to get file name for cert path")?, - ); + if let Some(host_cert_path) = as_cert_path { + let guest_cert_path = host_cert_path_to_target_path(&host_cert_path, true, false)?; scp_files.push((host_cert_path, guest_cert_path.clone())); cmd.push("--as-cert-path".to_string()); @@ -128,23 +162,10 @@ impl Applications { )?; Ok(None) } else { - if let Some(as_cert_path_str) = as_cert_path { - let cert_path = Path::new(as_cert_path_str); - if !cert_path.exists() { - anyhow::bail!("Certificate path does not exist: {}", cert_path.display()); - } - if !cert_path.is_file() { - anyhow::bail!("Certificate path is not a file: {}", cert_path.display()); - } - let docker_cert_path = Docker::get_docker_path(cert_path)?; + if let Some(host_cert_path) = as_cert_path { + let docker_cert_path = host_cert_path_to_target_path(&host_cert_path, false, true)?; cmd.push("--as-cert-path".to_string()); - let docker_cert_path_str = docker_cert_path.to_str().ok_or_else(|| { - anyhow::anyhow!( - "Docker path for certificate is not valid UTF-8: {}", - docker_cert_path.display() - ) - })?; - cmd.push(docker_cert_path_str.to_string()); + cmd.push(docker_cert_path.display().to_string()); } let workdir = Path::new(DOCKER_ACCLESS_CODE_MOUNT_DIR).join("applications"); let workdir_str = workdir.to_str().ok_or_else(|| { @@ -156,7 +177,7 @@ impl Applications { pub fn run( app_type: ApplicationType, - app_name: Functions, + app_name: ApplicationName, in_cvm: bool, as_url: Option, as_cert_path: Option, @@ -177,9 +198,13 @@ impl Applications { cmd.push(as_url.to_string()); } - if let Some(as_cert_path) = as_cert_path { + if let Some(host_cert_path) = as_cert_path { cmd.push("--as-cert-path".to_string()); - cmd.push(format!("{}", as_cert_path.display())); + cmd.push( + host_cert_path_to_target_path(&host_cert_path, true, false)? + .display() + .to_string(), + ); } // We don't need to SCP any files here, because we assume that the certificates @@ -189,16 +214,17 @@ impl Applications { Ok(None) } else { + let dir_name = match app_type { + ApplicationType::Function => "functions", + ApplicationType::Test => "test", + }; // Path matches CMake build directory: // ./applications/build-natie/{functions,test,workflows}/{name}/{binary_name} - let binary_path = match app_type { - ApplicationType::Function => Path::new(DOCKER_ACCLESS_CODE_MOUNT_DIR) - .join("applications/build-native") - // Note that we need to make the plural. - .join(format!("{app_type}s")) - .join(format!("{app_name}")) - .join(format!("{app_name}")), - }; + let binary_path = Path::new(DOCKER_ACCLESS_CODE_MOUNT_DIR) + .join("applications/build-native") + .join(dir_name) + .join(format!("{app_name}")) + .join(format!("{app_name}")); let binary_path_str = binary_path.to_str().ok_or_else(|| { anyhow::anyhow!("Binary path is not valid UTF-8: {}", binary_path.display()) @@ -206,19 +232,17 @@ impl Applications { let cmd = vec![binary_path_str.to_string()]; let as_env_vars: Vec = match (as_url, as_cert_path) { - (Some(as_url), Some(as_cert_path)) => attestation_service::get_as_env_vars( - &as_url, - as_cert_path.to_str().ok_or_else(|| { - anyhow::anyhow!( - "as cert path is not valid UTF-8 (path={})", - as_cert_path.display() - ) - })?, - ), + (Some(as_url), Some(host_cert_path)) => { + let docker_cert_path = + host_cert_path_to_target_path(&host_cert_path, false, true)? + .display() + .to_string(); + attestation_service::get_as_env_vars(&as_url, &docker_cert_path) + } _ => vec![], }; - Docker::run(&cmd, true, None, &as_env_vars, false, false) + Docker::run(&cmd, true, None, &as_env_vars, true, false) } } } diff --git a/accli/src/tasks/cvm.rs b/accli/src/tasks/cvm.rs index 17f6769..57eb6d9 100644 --- a/accli/src/tasks/cvm.rs +++ b/accli/src/tasks/cvm.rs @@ -6,7 +6,7 @@ use anyhow::{Context, Result}; use log::{debug, error, info, warn}; use std::{ io::{BufRead, BufReader, Read}, - path::PathBuf, + path::{Path, PathBuf}, process::{Child, Command, Stdio}, str::FromStr, sync::mpsc, @@ -99,6 +99,42 @@ fn snp_output_dir() -> PathBuf { path } +/// Remap a host path to a path in the cVM. +/// +/// This function takes a host path that must be within Accless' root, and +/// generates the same path inside the cVM's root filesystem. +pub fn remap_to_cvm_path(host_path: &Path) -> Result { + let absolute_host_path = if host_path.is_absolute() { + host_path.to_path_buf() + } else { + std::env::current_dir()?.join(host_path) + }; + let absolute_host_path = absolute_host_path.canonicalize().map_err(|e| { + let reason = format!( + "error canonicalizing path (path={}, error={})", + host_path.display(), + e + ); + error!("remap_to_cvm_path(): {reason}"); + anyhow::anyhow!(reason) + })?; + + let proj_root = Env::proj_root(); + if absolute_host_path.starts_with(&proj_root) { + let relative_path = absolute_host_path.strip_prefix(&proj_root).unwrap(); + let cvm_path = Path::new(CVM_ACCLESS_ROOT).join(relative_path); + Ok(cvm_path) + } else { + let reason = format!( + "path is outside the project root directory (path={}, root={})", + absolute_host_path.display(), + proj_root.display() + ); + error!("remap_to_cvm_path(): {reason}"); + anyhow::bail!(reason); + } +} + /// Helper method to read the logs from the cVM's stdout until it is ready. fn wait_for_cvm_ready(reader: R, timeout: Duration) -> Result<()> { let mut reader = BufReader::new(reader); diff --git a/accli/src/tasks/docker.rs b/accli/src/tasks/docker.rs index 40999c9..d730136 100644 --- a/accli/src/tasks/docker.rs +++ b/accli/src/tasks/docker.rs @@ -1,4 +1,5 @@ use crate::env::Env; +use anyhow::Result; use clap::ValueEnum; use log::error; use std::{ @@ -75,7 +76,7 @@ impl Docker { /// - `Err(anyhow::Error)`: An error if: /// - The path does not exist or cannot be canonicalized. /// - The path is outside the project's root directory. - pub fn get_docker_path(host_path: &Path) -> anyhow::Result { + pub fn remap_to_docker_path(host_path: &Path) -> Result { let absolute_host_path = if host_path.is_absolute() { host_path.to_path_buf() } else { From d74fdb573aaaba703aaf860028888a7d38879ace Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Wed, 26 Nov 2025 11:46:09 +0000 Subject: [PATCH 40/52] [applications] B: Use Mock Values From Mock Header --- applications/build.py | 2 +- applications/test/att-client-sgx/main.cpp | 14 ++++++-------- applications/test/att-client-snp/main.cpp | 14 ++++++-------- 3 files changed, 13 insertions(+), 17 deletions(-) diff --git a/applications/build.py b/applications/build.py index 5ed306c..1821da7 100644 --- a/applications/build.py +++ b/applications/build.py @@ -71,7 +71,7 @@ def compile(wasm=False, native=False, debug=False, clean=False, as_cert_path=Non parser.add_argument( "--debug", action="store_true", help="Build in debug mode." ) - parser.add_argument("--cert-path", type=str, help="Path to certificate PEM file.") + parser.add_argument("--as-cert-path", type=str, help="Path to certificate PEM file.") args = parser.parse_args() if args.clean: diff --git a/applications/test/att-client-sgx/main.cpp b/applications/test/att-client-sgx/main.cpp index 9831f99..8b4e82e 100644 --- a/applications/test/att-client-sgx/main.cpp +++ b/applications/test/att-client-sgx/main.cpp @@ -1,9 +1,12 @@ #include "accless/abe4/abe4.h" #include "accless/attestation/attestation.h" +#include "accless/attestation/mock.h" #include "accless/jwt/jwt.h" #include +using namespace accless::attestation::mock; + int main() { std::cout << "att-client-sgx: running test..." << std::endl; @@ -15,14 +18,9 @@ int main() { std::cout << "att-client-sgx: packed partial MPK into full MPK" << std::endl; - // These values are hard-coded in the mock SGX library in: - // `accless/libs/attestation/mock_sgx.cpp`. - std::string gid = "baz"; - std::string wfId = "foo"; - std::string nodeId = "bar"; - // The labels `wf` and `node` are hard-coded in the attestation-service. - std::string policy = id + ".wf:" + wfId + " & " + id + ".node:" + nodeId; + std::string policy = + id + ".wf:" + MOCK_WORKFLOW_ID + " & " + id + ".node:" + MOCK_NODE_ID; // Generate a test ciphertext that only us, after a succesful attestation, // should be able to decrypt. @@ -67,7 +65,7 @@ int main() { std::string uskB64 = accless::abe4::packFullKey({id}, {partialUskB64}); std::optional decrypted_gt = - accless::abe4::decrypt(uskB64, gid, policy, ct); + accless::abe4::decrypt(uskB64, MOCK_GID, policy, ct); if (!decrypted_gt.has_value()) { std::cerr << "att-client-sgx: CP-ABE decryption failed" diff --git a/applications/test/att-client-snp/main.cpp b/applications/test/att-client-snp/main.cpp index 9f9a9be..5e84e30 100644 --- a/applications/test/att-client-snp/main.cpp +++ b/applications/test/att-client-snp/main.cpp @@ -1,9 +1,12 @@ #include "accless/abe4/abe4.h" #include "accless/attestation/attestation.h" +#include "accless/attestation/mock.h" #include "accless/jwt/jwt.h" #include +using namespace accless::attestation::mock; + int main() { std::cout << "att-client-snp: running test..." << std::endl; @@ -15,14 +18,9 @@ int main() { std::cout << "att-client-snp: packed partial MPK into full MPK" << std::endl; - // These values are hard-coded in the mock SGX library in: - // `accless/libs/attestation/mock.cpp`. - std::string gid = "baz"; - std::string wfId = "foo"; - std::string nodeId = "bar"; - // The labels `wf` and `node` are hard-coded in the attestation-service. - std::string policy = id + ".wf:" + wfId + " & " + id + ".node:" + nodeId; + std::string policy = + id + ".wf:" + MOCK_WORKFLOW_ID + " & " + id + ".node:" + MOCK_NODE_ID; // Generate a test ciphertext that only us, after a succesful attestation, // should be able to decrypt. @@ -67,7 +65,7 @@ int main() { std::string uskB64 = accless::abe4::packFullKey({id}, {partialUskB64}); std::optional decrypted_gt = - accless::abe4::decrypt(uskB64, gid, policy, ct); + accless::abe4::decrypt(uskB64, MOCK_GID, policy, ct); if (!decrypted_gt.has_value()) { std::cerr << "att-client-snp: CP-ABE decryption failed" From d46213cc1ac0832688f865fcb7f59b5ffa5d42bb Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Wed, 26 Nov 2025 11:46:44 +0000 Subject: [PATCH 41/52] [config] E: Avoid Expensive Chowns On Entry --- config/docker/accless-experiments.dockerfile | 14 ++++++-- scripts/docker-entrypoint.sh | 35 ++++++++++++-------- 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/config/docker/accless-experiments.dockerfile b/config/docker/accless-experiments.dockerfile index 2e31a49..b68d6b7 100644 --- a/config/docker/accless-experiments.dockerfile +++ b/config/docker/accless-experiments.dockerfile @@ -1,8 +1,10 @@ FROM ghcr.io/faasm/cpp-sysroot:0.8.0 # Install rust -RUN rm -rf /root/.rustup \ - && apt update && apt install -y --no-install-recommends \ +ENV RUSTUP_HOME=/opt/rust/rustup +ENV CARGO_HOME=/opt/rust/cargo +ENV PATH=/opt/rust/cargo/bin:$PATH +RUN apt update && apt install -y --no-install-recommends \ build-essential \ curl \ gosu \ @@ -11,7 +13,13 @@ RUN rm -rf /root/.rustup \ wget \ zlib1g-dev \ && curl --proto '=https' --tlsv1.3 https://sh.rustup.rs -sSf | sh -s -- -y \ - && . "$HOME/.cargo/env" + # Create a group that owns the rust toolchain so that we can share it with + # our user at run time. + && groupadd -r rusttool \ + && mkdir -p /opt/rust/rustup /opt/rust/cargo \ + && chown -R root:rusttool /opt/rust \ + && chmod -R g+rwX /opt/rust \ + && find /opt/rust -type d -exec chmod g+s {} + # Deps for Azure's cVM guest library: OpenSSL + libcurl + TPM2-TSS. # The versions are taken from the pre-requisite script in the repo, and the diff --git a/scripts/docker-entrypoint.sh b/scripts/docker-entrypoint.sh index 27984ee..c9ec4f1 100755 --- a/scripts/docker-entrypoint.sh +++ b/scripts/docker-entrypoint.sh @@ -4,24 +4,33 @@ set -e USER_ID=${HOST_UID:-9001} GROUP_ID=${HOST_GID:-9001} +USER_NAME=accless # Group ID that owns /dev/sev-guest for SNP deployments (if present). SEV_GID=${SEV_GID:-} -groupadd -g $GROUP_ID accless -useradd -u $USER_ID -g $GROUP_ID -s /bin/bash -K UID_MAX=200000 accless -usermod -aG sudo accless -echo "accless ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/accless +# Create group if it doesn't exist +if ! getent group "$GROUP_ID" >/dev/null 2>&1; then + groupadd -g "$GROUP_ID" "$USER_NAME" +fi + +# Create user if it doesn't exist +if ! id -u "$USER_NAME" >/dev/null 2>&1; then + useradd -u "$USER_ID" -g "$GROUP_ID" -s /bin/bash -K UID_MAX=200000 -m "$USER_NAME" + usermod -aG sudo ${USER_NAME} + echo "$USER_NAME ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/$USER_NAME +fi -export HOME=/home/accless +export HOME=/home/${USER_NAME} mkdir -p ${HOME} -cp -r /root/.cargo ${HOME}/.cargo -cp -r /root/.rustup ${HOME}/.rustup -chown -R accless:accless /code -chown -R accless:accless ${HOME} -echo ". /code/accless/scripts/workon.sh" >> ${HOME}/.bashrc -echo ". ${HOME}/.cargo/env" >> ${HOME}/.bashrc +# Add user to group that owns the rust toolchain. +if getent group rusttool >/dev/null 2>&1; then + usermod -aG rusttool "$USER_NAME" +fi + +[ ! -e "$HOME/.cargo" ] && ln -s /opt/rust/cargo "$HOME/.cargo" +[ ! -e "$HOME/.rustup" ] && ln -s /opt/rust/rustup "$HOME/.rustup" # Add /dev/sev-guest owning group if necessary. if [ -e /dev/sev-guest ]; then @@ -32,12 +41,12 @@ if [ -e /dev/sev-guest ]; then fi # Add accless to that group (by GID to be robust to name differences) - usermod -aG "$SEV_GID" accless || true + usermod -aG "$SEV_GID" ${USER_NAME} || true else echo "WARNING: /dev/sev-guest present but SEV_GID not set!" fi fi -exec /usr/sbin/gosu accless bash -lc \ +exec /usr/sbin/gosu ${USER_NAME} bash -c \ 'source /code/accless/scripts/workon.sh; source "$HOME/.cargo/env"; exec "$@"' \ bash "$@" From 663865ed4cea1a308479f482272adc212bf4a519 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Wed, 26 Nov 2025 11:48:24 +0000 Subject: [PATCH 42/52] [build] E: Bump Minor Version To 0.9.0 --- Cargo.lock | 18 +++++++++--------- Cargo.toml | 2 +- VERSION | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4e03522..12d9056 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,7 +4,7 @@ version = 4 [[package]] name = "abe4" -version = "0.8.1" +version = "0.9.0" dependencies = [ "anyhow", "ark-bls12-381", @@ -22,7 +22,7 @@ dependencies = [ [[package]] name = "accless-finra-cloudevent-handler" -version = "0.8.1" +version = "0.9.0" dependencies = [ "cloudevents-sdk", "futures-util", @@ -38,7 +38,7 @@ dependencies = [ [[package]] name = "accless-ml-inference-cloudevent-handler" -version = "0.8.1" +version = "0.9.0" dependencies = [ "cloudevents-sdk", "futures-util", @@ -54,7 +54,7 @@ dependencies = [ [[package]] name = "accless-ml-training-cloudevent-handler" -version = "0.8.1" +version = "0.9.0" dependencies = [ "cloudevents-sdk", "futures-util", @@ -70,7 +70,7 @@ dependencies = [ [[package]] name = "accless-word-count-cloudevent-handler" -version = "0.8.1" +version = "0.9.0" dependencies = [ "cloudevents-sdk", "futures-util", @@ -86,7 +86,7 @@ dependencies = [ [[package]] name = "accli" -version = "0.8.1" +version = "0.9.0" dependencies = [ "anyhow", "base64 0.22.1", @@ -448,7 +448,7 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "attestation-service" -version = "0.8.1" +version = "0.9.0" dependencies = [ "abe4", "accli", @@ -2503,7 +2503,7 @@ dependencies = [ [[package]] name = "jwt" -version = "0.8.1" +version = "0.9.0" dependencies = [ "base64 0.22.1", "rsa", @@ -4269,7 +4269,7 @@ dependencies = [ [[package]] name = "template-graph" -version = "0.8.1" +version = "0.9.0" dependencies = [ "abe4", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index 8cb0162..d84e9e0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ members = [ ] [workspace.package] -version = "0.8.1" +version = "0.9.0" license-file = "LICENSE" authors = ["Large-Scale Data & Systems Group - Imperial College London"] edition = "2024" diff --git a/VERSION b/VERSION index 6f4eebd..ac39a10 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.8.1 +0.9.0 From bfb4e6790ef2c7aa6866ff4a30661052ed2c3cf1 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Wed, 26 Nov 2025 12:05:46 +0000 Subject: [PATCH 43/52] [scripts] E: Do Not Pin Code Branch In cVM --- .github/workflows/snp.yml | 13 ++++++++++++- scripts/snp/setup.sh | 6 +----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/.github/workflows/snp.yml b/.github/workflows/snp.yml index a651910..c0cef6c 100644 --- a/.github/workflows/snp.yml +++ b/.github/workflows/snp.yml @@ -37,10 +37,21 @@ jobs: if: steps.cache-snp.outputs.cache-hit != 'true' run: ./scripts/accli_wrapper.sh dev cvm setup --clean - # This step may take a while. - name: "Start attestation service in the background" run: ./scripts/accli_wrapper.sh attestation-service run --background --certs-dir ./certs --force-clean-certs --rebuild + # Fetch latest version of the code in the cVM. + - name: "Fetch code in the cVM" + run: | + # Work-out current branch name. + if [ -n "${{ github.head_ref }}" ]; then + BRANCH=${{ github.head_ref }} + else + BRANCH=${GITHUB_REF_NAME} + fi + ./scripts/accli_wrapper.sh dev cvm run \ + "git fetch origin $BRANCH && git checkout $BRANCH && git reset --hard origin/$BRANCH" + # Build SNP applications and embed the attestation service's certificate. - name: "Build SNP applications" run: ./scripts/accli_wrapper.sh applications build --clean --as-cert-path ./certs/cert.pem --in-cvm diff --git a/scripts/snp/setup.sh b/scripts/snp/setup.sh index d222b68..550ffed 100755 --- a/scripts/snp/setup.sh +++ b/scripts/snp/setup.sh @@ -234,21 +234,17 @@ systemctl enable ssh > /dev/null 2>&1 || systemctl enable ssh.service > /dev/nul echo "[provision/chroot] Installing rustup for ubuntu..." su -l ubuntu -c 'curl --proto "=https" --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y > /dev/null 2>&1' || true -# FIXME: remove branch name. echo "[provision/chroot] Cloning Accless repo (idempotent)..." su -l ubuntu -c ' cd /home/ubuntu && if [ ! -d accless/.git ]; then - git clone -b feature-escrow-func https://github.com/faasm/tless.git accless > /dev/null 2>&1; + git clone https://github.com/faasm/tless.git accless > /dev/null 2>&1; else echo "accless repo already present, skipping clone"; fi cd /home/ubuntu/accless && cargo build -p accli --release ' || true -# (Optional) You *could* pull docker images here, but it's messy because dockerd -# isn't running inside the chroot. Leaving that for runtime (cloud-init or first use). - echo "[provision/chroot] Provisioning done." EOF From 32088370a9289f0a7ffee411e948d8556b5e427c Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Wed, 26 Nov 2025 14:17:49 +0000 Subject: [PATCH 44/52] [scripts] B: Fix Code Ownership In Ctr --- config/docker/accless-experiments.dockerfile | 28 +++++++++++++------ .../entrypoint.sh} | 12 ++++++-- scripts/snp/cloud-init/user-data.in | 4 +-- 3 files changed, 30 insertions(+), 14 deletions(-) rename scripts/{docker-entrypoint.sh => docker/entrypoint.sh} (83%) diff --git a/config/docker/accless-experiments.dockerfile b/config/docker/accless-experiments.dockerfile index b68d6b7..4cb7b99 100644 --- a/config/docker/accless-experiments.dockerfile +++ b/config/docker/accless-experiments.dockerfile @@ -5,6 +5,7 @@ ENV RUSTUP_HOME=/opt/rust/rustup ENV CARGO_HOME=/opt/rust/cargo ENV PATH=/opt/rust/cargo/bin:$PATH RUN apt update && apt install -y --no-install-recommends \ + acl \ build-essential \ curl \ gosu \ @@ -67,23 +68,34 @@ RUN wget https://www.openssl.org/source/openssl-3.3.2.tar.gz \ && rm -rf /opt/tpm2-tss # Build specific libraries we need +ARG EXAMPLES_DIR=/code/faasm-examples RUN rm -rf /code \ && mkdir -p /code \ && cd /code \ # Checkout to examples repo to a specific commit - && git clone https://github.com/faasm/examples /code/faasm-examples \ - && cd /code/faasm-examples \ + && git clone https://github.com/faasm/examples ${EXAMPLES_DIR} \ + && cd ${EXAMPLES_DIR} \ && git checkout 3cd09e9cf41979fe73c8a9417b661ba08b5b3a75 \ && git submodule update --init -f cpp \ # Build specific CPP libs - && cd /code/faasm-examples/cpp \ + && cd ${EXAMPLES_DIR}/cpp \ && ./bin/inv_wrapper.sh libfaasm --clean \ && git submodule update --init ./third-party/zlib \ && ./bin/inv_wrapper.sh zlib \ - && cd /code/faasm-examples \ + && cd ${EXAMPLES_DIR} \ && git submodule update --init ./examples/opencv \ && ./bin/inv_wrapper.sh \ - opencv opencv --native + opencv opencv --native \ + # Add shared group ownership to faasm code. + && groupadd -r faasm \ + && mkdir -p ${EXAMPLES_DIR} \ + # maybe /code needs it too TODO delete me + && chown -R root:faasm ${EXAMPLES_DIR} \ + && chmod -R g+rwX ${EXAMPLES_DIR} \ + # make directories setgid so new stuff inherits the group TODO delete me + && find /code -type d -exec chmod g+s {} + \ + && setfacl -R -m g:faasm:rwX ${EXAMPLES_DIR} \ + && setfacl -R -d -m g:faasm:rwX ${EXAMPLES_DIR} # Prepare repository structure ARG ACCLESS_VERSION @@ -102,8 +114,8 @@ ENV ACCLESS_DOCKER=on # && python3 ./workflows/build.py WORKDIR /code/accless -COPY scripts/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh -RUN chmod +x /usr/local/bin/docker-entrypoint.sh -ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] +COPY scripts/docker/entrypoint.sh /usr/local/bin/docker_entrypoint.sh +RUN chmod +x /usr/local/bin/docker_entrypoint.sh +ENTRYPOINT ["/usr/local/bin/docker_entrypoint.sh"] CMD ["/bin/bash", "-l"] diff --git a/scripts/docker-entrypoint.sh b/scripts/docker/entrypoint.sh similarity index 83% rename from scripts/docker-entrypoint.sh rename to scripts/docker/entrypoint.sh index c9ec4f1..3e13790 100755 --- a/scripts/docker-entrypoint.sh +++ b/scripts/docker/entrypoint.sh @@ -29,6 +29,11 @@ if getent group rusttool >/dev/null 2>&1; then usermod -aG rusttool "$USER_NAME" fi +# Add user to group that owns the faasm toolchain. +if getent group faasm >/dev/null 2>&1; then + usermod -aG faasm "$USER_NAME" +fi + [ ! -e "$HOME/.cargo" ] && ln -s /opt/rust/cargo "$HOME/.cargo" [ ! -e "$HOME/.rustup" ] && ln -s /opt/rust/rustup "$HOME/.rustup" @@ -47,6 +52,7 @@ if [ -e /dev/sev-guest ]; then fi fi -exec /usr/sbin/gosu ${USER_NAME} bash -c \ - 'source /code/accless/scripts/workon.sh; source "$HOME/.cargo/env"; exec "$@"' \ - bash "$@" +echo ". /code/accless/scripts/workon.sh" >> ${HOME}/.bashrc +echo ". ${HOME}/.cargo/env" >> ${HOME}/.bashrc + +exec /usr/sbin/gosu ${USER_NAME} "$@" diff --git a/scripts/snp/cloud-init/user-data.in b/scripts/snp/cloud-init/user-data.in index 0cb9871..e0b4a63 100644 --- a/scripts/snp/cloud-init/user-data.in +++ b/scripts/snp/cloud-init/user-data.in @@ -20,7 +20,7 @@ users: - name: ubuntu sudo: ALL=(ALL) NOPASSWD:ALL - ssh-authorized-keys: + ssh_authorized_keys: - "${SSH_PUB_KEY}" # Very light runtime tweaks only. @@ -38,7 +38,5 @@ runcmd: - [ systemctl, enable, --now, ssh.service ] # Optional: pull docker image at runtime (idempotent). - [ sudo, "-u", "ubuntu", "bash", "-lc", "docker pull ghcr.io/faasm/accless-experiments:${ACCLESS_VERSION} || true" ] - # Explicit readiness marker on the serial console (for your Rust QEMU wrapper). - - [ sh, -c, 'logger -t cloud-init "Accless SNP test instance v${ACCLESS_VERSION} is ready."' ] final_message: "Accless SNP test instance v${ACCLESS_VERSION} is ready." From 7bfc09e5dd6d2ae238d1d44def9bba70f15a8bad Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Wed, 26 Nov 2025 14:23:12 +0000 Subject: [PATCH 45/52] [accli] B: Fix Persistance In cVMs --- accli/src/tasks/cvm.rs | 181 +++++++++++-------- accli/src/tasks/docker.rs | 5 +- config/docker/accless-experiments.dockerfile | 2 +- 3 files changed, 114 insertions(+), 74 deletions(-) diff --git a/accli/src/tasks/cvm.rs b/accli/src/tasks/cvm.rs index 57eb6d9..2732a14 100644 --- a/accli/src/tasks/cvm.rs +++ b/accli/src/tasks/cvm.rs @@ -77,14 +77,9 @@ struct QemuGuard { child: Child, } -impl Drop for QemuGuard { - fn drop(&mut self) { - info!("Killing QEMU process with PID: {}", self.child.id()); - if let Err(e) = self.child.kill() { - error!("Failed to kill QEMU process: {}", e); - } - } -} +// =============================================================================================== +// Helper Functions +// =============================================================================================== fn snp_root() -> PathBuf { let mut path = Env::proj_root(); @@ -99,42 +94,6 @@ fn snp_output_dir() -> PathBuf { path } -/// Remap a host path to a path in the cVM. -/// -/// This function takes a host path that must be within Accless' root, and -/// generates the same path inside the cVM's root filesystem. -pub fn remap_to_cvm_path(host_path: &Path) -> Result { - let absolute_host_path = if host_path.is_absolute() { - host_path.to_path_buf() - } else { - std::env::current_dir()?.join(host_path) - }; - let absolute_host_path = absolute_host_path.canonicalize().map_err(|e| { - let reason = format!( - "error canonicalizing path (path={}, error={})", - host_path.display(), - e - ); - error!("remap_to_cvm_path(): {reason}"); - anyhow::anyhow!(reason) - })?; - - let proj_root = Env::proj_root(); - if absolute_host_path.starts_with(&proj_root) { - let relative_path = absolute_host_path.strip_prefix(&proj_root).unwrap(); - let cvm_path = Path::new(CVM_ACCLESS_ROOT).join(relative_path); - Ok(cvm_path) - } else { - let reason = format!( - "path is outside the project root directory (path={}, root={})", - absolute_host_path.display(), - proj_root.display() - ); - error!("remap_to_cvm_path(): {reason}"); - anyhow::bail!(reason); - } -} - /// Helper method to read the logs from the cVM's stdout until it is ready. fn wait_for_cvm_ready(reader: R, timeout: Duration) -> Result<()> { let mut reader = BufReader::new(reader); @@ -180,6 +139,84 @@ fn wait_for_cvm_ready(reader: R, timeout: Durat } } +fn set_ssh_options(cmd: &mut Command) { + cmd.stderr(Stdio::null()) + .arg("-p") + .arg(SSH_PORT.to_string()) + .arg("-i") + .arg(format!("{}/{EPH_PRIVKEY}", snp_output_dir().display())) + .arg("-o") + .arg("StrictHostKeyChecking=no") + .arg("-o") + .arg("UserKnownHostsFile=/dev/null") + .arg(format!("{CVM_USER}@localhost")); +} + +fn poweroff_vm() -> Result<()> { + let mut cmd = Command::new("ssh"); + set_ssh_options(&mut cmd); + let status = cmd.args(["sudo", "shutdown", "now"]).status()?; + if !status.success() { + anyhow::bail!("poweroff_vm(): shutting down failed"); + } + + Ok(()) +} + +impl Drop for QemuGuard { + fn drop(&mut self) { + if let Err(e) = poweroff_vm() { + warn!("drop(): shutting down VM cleanly failed (error={e:?})"); + if let Err(e) = self.child.kill() { + error!("Failed to kill QEMU process (error={e:?})"); + } + } else if let Err(e) = self.child.wait() { + error!("drop(): error waiting for child process to finish (error={e:?})"); + } + } +} + +// =============================================================================================== +// Public API +// =============================================================================================== + +/// Remap a host path to a path in the cVM. +/// +/// This function takes a host path that must be within Accless' root, and +/// generates the same path inside the cVM's root filesystem. +pub fn remap_to_cvm_path(host_path: &Path) -> Result { + let absolute_host_path = if host_path.is_absolute() { + host_path.to_path_buf() + } else { + std::env::current_dir()?.join(host_path) + }; + let absolute_host_path = absolute_host_path.canonicalize().map_err(|e| { + let reason = format!( + "error canonicalizing path (path={}, error={})", + host_path.display(), + e + ); + error!("remap_to_cvm_path(): {reason}"); + anyhow::anyhow!(reason) + })?; + + let proj_root = Env::proj_root(); + if absolute_host_path.starts_with(&proj_root) { + let relative_path = absolute_host_path.strip_prefix(&proj_root).unwrap(); + let cvm_path = Path::new(CVM_ACCLESS_ROOT).join(relative_path); + Ok(cvm_path) + } else { + let reason = format!( + "path is outside the project root directory (path={}, root={})", + absolute_host_path.display(), + proj_root.display() + ); + error!("remap_to_cvm_path(): {reason}"); + anyhow::bail!(reason); + } +} + +/// Build the cVM Image. pub fn build(clean: bool, component: Option) -> Result<()> { info!("build(): building cVM image..."); let mut cmd = Command::new(format!("{}/setup.sh", snp_root().display())); @@ -216,6 +253,9 @@ pub fn build(clean: bool, component: Option) -> Result<()> { /// `HostPath` is the path to the file on the host machine, and `GuestPath` is /// the relative path inside the cVM. The `GuestPath` will automatically be /// prefixed with `/home/ubuntu/accless`. +/// - `cwd`: An optional `PathBuf` representing the working directory inside the +/// cVM, relative to `/home/ubuntu/accless`. If provided, the command will be +/// executed in this directory. /// /// # Returns /// @@ -241,9 +281,6 @@ pub fn build(clean: bool, component: Option) -> Result<()> { /// ) /// .unwrap(); /// ``` -/// - `cwd`: An optional `PathBuf` representing the working directory inside the -/// cVM, relative to `/home/ubuntu/accless`. If provided, the command will be -/// executed in this directory. pub fn run( cmd: &[String], scp_files: Option<&[(PathBuf, PathBuf)]>, @@ -274,13 +311,31 @@ pub fn run( if let Some(files) = scp_files { info!("run(): copying files into cVM..."); for (host_path, guest_path) in files { - let full_guest_path = format!("{}/{}", CVM_ACCLESS_ROOT, guest_path.display()); + let full_guest_path = if !guest_path.starts_with(CVM_ACCLESS_ROOT) { + format!("{}/{}", CVM_ACCLESS_ROOT, guest_path.display()) + } else { + guest_path.display().to_string() + }; info!( "run(): copying {} to {CVM_USER}@localhost:{}", host_path.display(), full_guest_path ); + // Make sure the directory we are copying to exists. + let mut ssh_mkdir_cmd = Command::new("ssh"); + set_ssh_options(&mut ssh_mkdir_cmd); + ssh_mkdir_cmd.args([ + "mkdir".to_string(), + "-p".to_string(), + guest_path.parent().unwrap().display().to_string(), + ]); + + let status = ssh_mkdir_cmd.status()?; + if !status.success() { + anyhow::bail!("run(): failed to mkdir inside cVM"); + } + let mut scp_cmd = Command::new("scp"); scp_cmd .arg("-P") @@ -322,17 +377,8 @@ pub fn run( final_cmd.join(" ") ); let mut ssh_cmd = Command::new("ssh"); - ssh_cmd - .arg("-p") - .arg(SSH_PORT.to_string()) - .arg("-i") - .arg(format!("{}/{EPH_PRIVKEY}", snp_output_dir().display())) - .arg("-o") - .arg("StrictHostKeyChecking=no") - .arg("-o") - .arg("UserKnownHostsFile=/dev/null") - .arg(format!("{CVM_USER}@localhost")) - .args(final_cmd); + set_ssh_options(&mut ssh_cmd); + ssh_cmd.args(final_cmd); let status = ssh_cmd.status()?; if !status.success() { @@ -373,19 +419,10 @@ pub fn cli(cwd: Option<&PathBuf>) -> Result<()> { interactive_cmd.push("bash".to_string()); // Start a bash shell info!("cli(): opening interactive SSH session to cVM"); - let status = Command::new("ssh") - .arg("-p") - .arg(SSH_PORT.to_string()) - .arg("-i") - .arg(format!("{}/{EPH_PRIVKEY}", snp_output_dir().display())) - .arg("-o") - .arg("StrictHostKeyChecking=no") - .arg("-o") - .arg("UserKnownHostsFile=/dev/null") - .arg("-t") // Allocate a pseudo-terminal - .arg(format!("{CVM_USER}@localhost")) - .args(interactive_cmd) - .status()?; + let mut cmd = Command::new("ssh"); + cmd.arg("-t"); + set_ssh_options(&mut cmd); + let status = cmd.args(interactive_cmd).status()?; if !status.success() { anyhow::bail!("cli(): interactive SSH session failed"); diff --git a/accli/src/tasks/docker.rs b/accli/src/tasks/docker.rs index d730136..057942d 100644 --- a/accli/src/tasks/docker.rs +++ b/accli/src/tasks/docker.rs @@ -52,6 +52,7 @@ pub const DOCKER_ACCLESS_CODE_MOUNT_DIR: &str = "/code/accless"; impl Docker { const ACCLESS_DEV_CONTAINER_NAME: &'static str = "accless-dev"; + const ACCLESS_DEV_CONTAINER_HOSTNAME: &'static str = "accless-ctr"; /// # Description /// @@ -263,7 +264,9 @@ impl Docker { .arg("run") .arg("--rm") .arg("--name") - .arg(Self::ACCLESS_DEV_CONTAINER_NAME); + .arg(Self::ACCLESS_DEV_CONTAINER_NAME) + .arg("--hostname") + .arg(Self::ACCLESS_DEV_CONTAINER_HOSTNAME); if interactive { run_cmd.arg("-it"); } diff --git a/config/docker/accless-experiments.dockerfile b/config/docker/accless-experiments.dockerfile index 4cb7b99..b818d0f 100644 --- a/config/docker/accless-experiments.dockerfile +++ b/config/docker/accless-experiments.dockerfile @@ -118,4 +118,4 @@ COPY scripts/docker/entrypoint.sh /usr/local/bin/docker_entrypoint.sh RUN chmod +x /usr/local/bin/docker_entrypoint.sh ENTRYPOINT ["/usr/local/bin/docker_entrypoint.sh"] -CMD ["/bin/bash", "-l"] +CMD ["/bin/bash", "-c"] From fe6373a000208d263af2acdcbf6a92387b43485a Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Wed, 26 Nov 2025 14:38:02 +0000 Subject: [PATCH 46/52] [docs] E: Update README --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 5110931..3d77f59 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,10 @@ Integration Tests + + SNP End-to-End Tests +

From 978c73775233cd8a9240a7bdeb1daa42d4cf3c5f Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Wed, 26 Nov 2025 14:38:39 +0000 Subject: [PATCH 47/52] [scripts] B: Make Sure Workon.sh Is Run In Ctr --- config/docker/accless-experiments.dockerfile | 2 +- scripts/docker/entrypoint.sh | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/config/docker/accless-experiments.dockerfile b/config/docker/accless-experiments.dockerfile index b818d0f..4cb7b99 100644 --- a/config/docker/accless-experiments.dockerfile +++ b/config/docker/accless-experiments.dockerfile @@ -118,4 +118,4 @@ COPY scripts/docker/entrypoint.sh /usr/local/bin/docker_entrypoint.sh RUN chmod +x /usr/local/bin/docker_entrypoint.sh ENTRYPOINT ["/usr/local/bin/docker_entrypoint.sh"] -CMD ["/bin/bash", "-c"] +CMD ["/bin/bash", "-l"] diff --git a/scripts/docker/entrypoint.sh b/scripts/docker/entrypoint.sh index 3e13790..0a145ba 100755 --- a/scripts/docker/entrypoint.sh +++ b/scripts/docker/entrypoint.sh @@ -55,4 +55,8 @@ fi echo ". /code/accless/scripts/workon.sh" >> ${HOME}/.bashrc echo ". ${HOME}/.cargo/env" >> ${HOME}/.bashrc -exec /usr/sbin/gosu ${USER_NAME} "$@" +exec /usr/sbin/gosu "$USER_NAME" bash -c \ + 'source /code/accless/scripts/workon.sh 2>/dev/null || true; \ + source "$HOME/.cargo/env" 2>/dev/null || true; \ + exec "$@"' \ + bash "$@" From 38b5ce98e90ffc3b95f975bf8bbed29ba0dbc905 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Wed, 26 Nov 2025 14:41:36 +0000 Subject: [PATCH 48/52] [accli] B: Create PID File Dirs In AS BG Spawn --- accli/src/tasks/attestation_service.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/accli/src/tasks/attestation_service.rs b/accli/src/tasks/attestation_service.rs index 9156be3..795924e 100644 --- a/accli/src/tasks/attestation_service.rs +++ b/accli/src/tasks/attestation_service.rs @@ -82,7 +82,9 @@ impl AttestationService { .stderr(Stdio::null()) .spawn() .context("Failed to spawn attestation service in background")?; - fs::write(PID_FILE_PATH, child.id().to_string()).context("Failed to write PID file")?; + fs::create_dir_all(PID_FILE_PATH).context("run(): failed to create PID file dirs")?; + fs::write(PID_FILE_PATH, child.id().to_string()) + .context("run(): failed to write PID file")?; info!("run(): attestation service spawned (PID={})", child.id()); Ok(()) } else { From 5f35f1b0881a03d9917fe5d5e0b0140618c860ef Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Thu, 27 Nov 2025 10:24:06 +0000 Subject: [PATCH 49/52] [accli] B: Fix Spawn Of Background AS --- .github/workflows/snp.yml | 4 ++-- accli/src/tasks/attestation_service.rs | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/snp.yml b/.github/workflows/snp.yml index c0cef6c..6aa3f7d 100644 --- a/.github/workflows/snp.yml +++ b/.github/workflows/snp.yml @@ -49,8 +49,8 @@ jobs: else BRANCH=${GITHUB_REF_NAME} fi - ./scripts/accli_wrapper.sh dev cvm run \ - "git fetch origin $BRANCH && git checkout $BRANCH && git reset --hard origin/$BRANCH" + ./scripts/accli_wrapper.sh dev cvm run -- \ + git fetch origin $BRANCH && git checkout $BRANCH && git reset --hard origin/$BRANCH # Build SNP applications and embed the attestation service's certificate. - name: "Build SNP applications" diff --git a/accli/src/tasks/attestation_service.rs b/accli/src/tasks/attestation_service.rs index 795924e..322c4ca 100644 --- a/accli/src/tasks/attestation_service.rs +++ b/accli/src/tasks/attestation_service.rs @@ -8,6 +8,7 @@ use nix::{ use reqwest; use std::{ fs, + path::Path, process::{Command, Stdio}, }; @@ -81,8 +82,10 @@ impl AttestationService { .stdout(Stdio::null()) .stderr(Stdio::null()) .spawn() - .context("Failed to spawn attestation service in background")?; - fs::create_dir_all(PID_FILE_PATH).context("run(): failed to create PID file dirs")?; + .context("run(): failed to spawn attestation service in background")?; + let pid_file_path = Path::new(PID_FILE_PATH); + fs::create_dir_all(pid_file_path.parent().unwrap()) + .context("run(): failed to create PID file dirs")?; fs::write(PID_FILE_PATH, child.id().to_string()) .context("run(): failed to write PID file")?; info!("run(): attestation service spawned (PID={})", child.id()); From b6946f19eab6af5e6c63997cdf0bbee22fb4b5c8 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Thu, 27 Nov 2025 12:24:32 +0000 Subject: [PATCH 50/52] [scripts] E: Check Host Reqs In SNP Setup --- accli/src/tasks/cvm.rs | 3 +++ scripts/snp/setup.sh | 35 ++++++++++++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/accli/src/tasks/cvm.rs b/accli/src/tasks/cvm.rs index 2732a14..1e41186 100644 --- a/accli/src/tasks/cvm.rs +++ b/accli/src/tasks/cvm.rs @@ -33,6 +33,7 @@ pub fn parse_host_guest_path(s: &str) -> anyhow::Result<(PathBuf, PathBuf)> { #[derive(Debug, Clone, Copy)] pub enum Component { + Check, Apt, Qemu, Ovmf, @@ -47,6 +48,7 @@ impl FromStr for Component { fn from_str(s: &str) -> Result { match s { + "check" => Ok(Component::Check), "apt" => Ok(Component::Apt), "qemu" => Ok(Component::Qemu), "ovmf" => Ok(Component::Ovmf), @@ -62,6 +64,7 @@ impl FromStr for Component { impl std::fmt::Display for Component { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { + Component::Check => write!(f, "check"), Component::Apt => write!(f, "apt"), Component::Qemu => write!(f, "qemu"), Component::Ovmf => write!(f, "ovmf"), diff --git a/scripts/snp/setup.sh b/scripts/snp/setup.sh index 550ffed..bfa98b5 100755 --- a/scripts/snp/setup.sh +++ b/scripts/snp/setup.sh @@ -16,7 +16,36 @@ clean() { mkdir -p ${OUTPUT_DIR} } -# TODO: check host kernel +# +# Check the host system is configured properly. +check_host_reqs() { + local device="/dev/sev" + local required_group="kvm" + local user="$(id -un)" + + print_info "Checking host system requirements..." + + # Check `/dev/sev` exists. + if [[ ! -e "$device" ]]; then + print_error "check_host_reqs(): $device does not exist" + exit 1 + fi + + # Check `/dev/sev` is owned by the `kvm` group. + device_group="$(stat -c "%G" "$device")" + if [[ "$device_group" != "$required_group" ]]; then + pritn_error "check_host_reqs(): $device is owned by group '$device_group', expected '$required_group'." + exit 1 + fi + + # Check calling user is in the `kvm` group. + if ! id -nG "$user" | grep -qw "$required_group"; then + print_error "check_host_reqs(): user '$user' is not in group '$required_group'." + exit 1 + fi + + print_success "check_host_reqs(): $device is owned by group '$required_group' and user '$user' is in that group." +} # # Fetch the linux kernel image. @@ -339,6 +368,9 @@ main() { if [[ -n "$component" ]]; then case "$component" in + check) + check_host_reqs + ;; apt) install_apt_deps ;; @@ -366,6 +398,7 @@ main() { ;; esac else + check_host_reqs install_apt_deps build_qemu build_ovmf From 8ca0c2cbe5da3613bb575eb9c62a841f3b40fb63 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Thu, 27 Nov 2025 12:28:11 +0000 Subject: [PATCH 51/52] [attestation-service] B: Properly Poll Certs --- attestation-service/tests/as_api_tests.rs | 26 +++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/attestation-service/tests/as_api_tests.rs b/attestation-service/tests/as_api_tests.rs index 89e45a2..12f1e90 100644 --- a/attestation-service/tests/as_api_tests.rs +++ b/attestation-service/tests/as_api_tests.rs @@ -7,10 +7,13 @@ use serial_test::serial; use std::{ path::{Path, PathBuf}, process::Stdio, - time::Duration, + time::{Duration, Instant}, }; use tempfile::tempdir; -use tokio::process::{Child, Command}; +use tokio::{ + process::{Child, Command}, + time::sleep, +}; struct ChildGuard(Child); @@ -127,6 +130,25 @@ async fn test_att_clients() -> Result<()> { let cert_path = get_public_certificate_path(&certs_dir); + // Wait until cert path to be ready. + let deadline = Instant::now() + Duration::from_secs(15); + let poll_interval = Duration::from_millis(100); + loop { + if cert_path.exists() { + break; + } + if Instant::now() >= deadline { + let reason = format!( + "timed-out waiting for certs to become available (path={})", + cert_path.display() + ); + error!("test_att_clients(): {reason}"); + anyhow::bail!(reason); + } + + sleep(poll_interval).await; + } + // While it is starting, rebuild the test application so that we can inject the // new certificates. Note that we need to pass the certificate's path // _inside_ the container, as application build happens inside the From 9100a3ef82fa60b5b50392009e4723c7b5274b92 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Thu, 27 Nov 2025 13:24:27 +0000 Subject: [PATCH 52/52] [ci] B: Fix Syntax Error In snp.yml --- .github/workflows/snp.yml | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/.github/workflows/snp.yml b/.github/workflows/snp.yml index 6aa3f7d..4b1dcc1 100644 --- a/.github/workflows/snp.yml +++ b/.github/workflows/snp.yml @@ -25,16 +25,7 @@ jobs: - name: "Check out the code" uses: actions/checkout@v4 - - name: "Cache SNP artefacts" - id: cache-snp - uses: actions/cache@v4 - with: - path: scripts/snp/output - key: snp-setup-${{ runner.os }}-${{ hashFiles('scripts/snp/**') }} - - # Only re-run on a cache miss (setup may take a while) - name: "Run SNP setup" - if: steps.cache-snp.outputs.cache-hit != 'true' run: ./scripts/accli_wrapper.sh dev cvm setup --clean - name: "Start attestation service in the background" @@ -50,7 +41,7 @@ jobs: BRANCH=${GITHUB_REF_NAME} fi ./scripts/accli_wrapper.sh dev cvm run -- \ - git fetch origin $BRANCH && git checkout $BRANCH && git reset --hard origin/$BRANCH + "git fetch origin $BRANCH && git checkout $BRANCH && git reset --hard origin/$BRANCH" # Build SNP applications and embed the attestation service's certificate. - name: "Build SNP applications" @@ -59,6 +50,10 @@ jobs: - name: "Run supported SNP applications" run: | # First get the external IP so that we can reach the attestation-service from the cVM. - AS_URL=$(accli attestation-service health --url "https://0.0.0.0:8443" --cert-path ./certs/cert.pem \ + AS_URL=$(./scripts/accli_wrapper.sh attestation-service health --url "https://0.0.0.0:8443" --cert-path ./certs/cert.pem 2>&1 \ | grep "attestation service is healthy and reachable on:" | awk '{print $NF}') - ./scripts/accli_wrapper.sh applications run function escrow-xput --as-url ${AS_URL} --as-cert-path ./certs/cert.pem + echo "Got AS URL: ${AS_URL}" + ./scripts/accli_wrapper.sh applications run function escrow-xput --as-url ${AS_URL} --as-cert-path ./certs/cert.pem --in-cvm + + - name: "Stop attestation service in the background" + run: ./scripts/accli_wrapper.sh attestation-service stop