feat(sgx): handle CSSA 0-3 with 2 sallyport blocks

CSSA 1 and 2 get their own block.
CSSA 0 and 3 don't need a block.

Normal block usage looks like this:
* Enclave CSSA 0 fills in registers and executes syscall op.
* Host sees exception, calls CSSA 1 with sallyport block 0.
  * Enclave CSSA 1 fills in sallyport block, executes syscall op.
  * Host sees exception, handles sallyport block 0 and calls CSSA 2.
    * Enclave CSSA 2 confirms CSSA 1 syscall op and increases instruction
      pointer of CSSA 1. Exits to the host.
    * Host sees clean EEXIT of CSSA 2, calls CSSA 1.
  * Enclave CSSA 1 continues after syscall op and handles sallyport block 0,
    then increases CSSA 0 instruction pointer and   exits to the host.
  * Host sees clean EEXIT of CSSA 1, calls CSSA 0.
* Enclave CSSA 0 handles result of syscall.

Now, if there was lazy memory allocation for CSSA 0, with page fault handling,
the following situation could occur: While enclave CSSA 1 handles the result
of sallyport block 0 and copies data back to the userspace memory,
a page fault could occur.
Therefore we need a second sallyport block,
which could be uses by CSSA 2 to handle the page fault occuring in CSSA 1.

This situation looks like this:
* Enclave CSSA 0 fills in registers and executes syscall op.
* Host sees exception, calls CSSA 1 with sallyport block 0.
  * Enclave CSSA 1 fills in sallyport block, executes syscall op.
  * Host sees exception, handles sallyport block 0 and calls CSSA 2.
    * Enclave CSSA 2 confirms CSSA 1 syscall op and increases instruction
      pointer of CSSA 1. Exits to the host.
    * Host sees clean EEXIT of CSSA 2, calls CSSA 1.
  * Enclave CSSA 1 continues after syscall op and handles sallyport block 0,
    and encountes a page fault for touching not yet mapped memory of CSSA 0.
  * Host sees page fault exception, calls CSSA 2.
    * CSSA 2 sees page fault exception, does some memory operations using
      sallyport block 1 and syscall op.
    * Host sees syscall exception, handles sallyport block 1 and calls CSSA 3.
      * Enclave CSSA 3 confirms CSSA 2 syscall op and increases instruction
        pointer of CSSA 2. Exits to the host.
      * Host sees clean EEXIT of CSSA 3, calls CSSA 2.
    * Enclave CSSA 2 continues after syscall op and handles sallyport block 1,
      then increases CSSA 1 instruction pointer and exits to the thost.
    * Host sees clean EEXIT of CSSA 2, calls CSSA 1.
  * Enclave CSSA 1 continues handling sallyport block 0, now that the page
  is correctly mapped, then increases CSSA 0 instruction pointer and exits
  to the host.
  * Host sees clean EEXIT of CSSA 1, calls CSSA 0.
* Enclave CSSA 0 handles result of syscall.

Signed-off-by: Harald Hoyer <harald@profian.com>

crates/shim-sgx/layout.ld

@@ -62,7 +62,7 @@ SECTIONS {
         . += 16;
         QUAD(ADDR(.enarx.ssa))   /* OSSA */
         LONG(0)                   /* CSSA */
-        LONG(3)                   /* NSSA */
+        LONG(4)                   /* NSSA */
         QUAD(_start)              /* OENTRY */
         . = ALIGN(4K);
     } :tcs =0

crates/shim-sgx/src/handler/mod.rs

@@ -439,7 +439,7 @@ impl<'a> Handler<'a> {
     }
 
     /// Finish handling an exception
-    pub fn finish(ssa: &'a mut StateSaveArea, block: &'a mut [usize], tcb: &'a mut Tcb) {
+    pub fn finish(ssa: &'a mut StateSaveArea, block: Option<&'a mut [usize]>, tcb: &'a mut Tcb) {
         if let Some(Vector::InvalidOpcode) = ssa.vector() {
             if let OP_SYSCALL | OP_CPUID = unsafe { read_unaligned(ssa.gpr.rip as _) } {
                 // Skip the instruction.
@@ -448,8 +448,10 @@ impl<'a> Handler<'a> {
             }
         }
 
-        let mut h = Self::new(ssa, block, tcb, 0);
-        h.print_ssa_stack_trace();
+        if let Some(block) = block {
+            let mut h = Self::new(ssa, block, tcb, 0);
+            h.print_ssa_stack_trace();
+        }
 
         unsafe { asm!("ud2", options(noreturn)) };
     }

crates/shim-sgx/src/main.rs

@@ -26,8 +26,14 @@ use enarx_shim_sgx::thread::{
 };
 use enarx_shim_sgx::{
     entry, handler, shim_address, ATTR, BLOCK_SIZE, CSSA_0_STACK_SIZE, ENARX_EXEC_START,
-    ENARX_SHIM_ADDRESS, ENCL_SIZE, ENCL_SIZE_BITS, MISC,
+    ENARX_SHIM_ADDRESS, ENCL_SIZE, ENCL_SIZE_BITS, MISC, NUM_SSA,
 };
+use noted::noted;
+use primordial::Page;
+use sallyport::util::ptr::is_aligned_non_null;
+use sallyport::{elf::note, REQUIRES};
+use sgx::parameters::{Attributes, MiscSelect};
+use sgx::ssa::{GenPurposeRegs, StateSaveArea};
 
 #[panic_handler]
 #[cfg(not(test))]
@@ -38,12 +44,6 @@ fn panic(_info: &core::panic::PanicInfo<'_>) -> ! {
 
 // ============== REAL CODE HERE ===============
 
-use noted::noted;
-use primordial::Page;
-use sallyport::{elf::note, REQUIRES};
-use sgx::parameters::{Attributes, MiscSelect};
-use sgx::ssa::{GenPurposeRegs, StateSaveArea};
-
 noted! {
     static NOTE_REQUIRES<note::NAME, note::REQUIRES, [u8; REQUIRES.len()]> = REQUIRES;
 
@@ -275,29 +275,35 @@ global_asm!(
         CSSA_0_STK_TCS_SZ = const CSSA_0_STACK_SIZE + Page::SIZE,
 );
 
-unsafe extern "C" fn main(
-    block: &mut [usize; BLOCK_SIZE / core::mem::size_of::<usize>()],
-    ssas: &mut [StateSaveArea; 3],
-    cssa: usize,
-    tcb: &mut MaybeUninit<Tcb>,
-) -> i32 {
-    // Enable exceptions:
-    ssas[cssa].extra[0] = 1;
+fn validate_block_ptr(ptr: *mut u8) -> &'static mut [usize; BLOCK_SIZE / size_of::<usize>()] {
+    is_aligned_non_null::<[usize; BLOCK_SIZE / size_of::<usize>()]>(ptr as usize).unwrap();
 
     // As host is a separate application from the shim, all the data coming from
     // it needs to be validated explicitly.  Thus, check that the Sallyport
     // block is outside the shim address space:
-    let block_start = block.as_ptr() as usize;
+    let block_start = ptr as usize;
     let block_end = block_start + BLOCK_SIZE;
     let shim_start = shim_address();
     let shim_end = shim_start + ENCL_SIZE;
 
     if (block_start >= shim_start && block_start < shim_end)
         || (block_end > shim_start && block_end <= shim_end)
     {
-        panic!();
+        panic!("Sallyport block is inside the shim address space");
     }
 
+    unsafe { &mut *(ptr as *mut [usize; BLOCK_SIZE / size_of::<usize>()]) }
+}
+
+unsafe extern "C" fn main(
+    block_ptr: *mut u8,
+    ssas: &mut [StateSaveArea; NUM_SSA],
+    cssa: usize,
+    tcb: &mut MaybeUninit<Tcb>,
+) -> i32 {
+    // Enable exceptions:
+    ssas[cssa].extra[0] = 1;
+
     let mut ret = 0;
 
     match cssa {
@@ -337,17 +343,24 @@ unsafe extern "C" fn main(
         1 => {
             // cssa == 0 already initialized the TCB
             let tcb = tcb.assume_init_mut();
+            let block = validate_block_ptr(block_ptr);
             handler::Handler::handle(
                 &mut ssas[0],
                 block.as_mut_slice(),
                 tcb,
                 _start as usize as _,
             )
         }
-        n => {
+        2 => {
+            let tcb = tcb.assume_init_mut();
+            let block = validate_block_ptr(block_ptr);
+            handler::Handler::finish(&mut ssas[1], Some(block.as_mut_slice()), tcb)
+        }
+        3 => {
             let tcb = tcb.assume_init_mut();
-            handler::Handler::finish(&mut ssas[n - 1], block.as_mut_slice(), tcb)
+            handler::Handler::finish(&mut ssas[2], None, tcb)
         }
+        _ => panic!("CSSA > 3"),
     }
 
     // Disable exceptions:

src/backend/sgx/thread.rs

@@ -6,12 +6,12 @@ use crate::backend::execute_gdb;
 use crate::backend::Command;
 
 use std::arch::asm;
-use std::io;
 use std::iter;
 use std::mem::{size_of, MaybeUninit};
 #[cfg(feature = "gdb")]
 use std::net::TcpStream;
 use std::sync::Arc;
+use std::{io, ptr};
 
 use anyhow::{bail, Context, Result};
 use sallyport::item;
@@ -25,7 +25,7 @@ pub struct Thread {
     keep: Arc<super::Keep>,
     vdso: &'static Symbol,
     tcs: super::Tcs,
-    block: Vec<usize>,
+    block: [Vec<usize>; 2],
     cssa: usize,
     how: usize,
     #[cfg(feature = "gdb")]
@@ -53,7 +53,10 @@ impl super::super::Keep for super::Keep {
             None => return Ok(None),
         };
 
-        let block = vec![0; self.sallyport_block_size as usize / size_of::<usize>()];
+        let block = [
+            vec![0; self.sallyport_block_size as usize / size_of::<usize>()],
+            vec![0; self.sallyport_block_size as usize / size_of::<usize>()],
+        ];
 
         Ok(Some(Box::new(Thread {
             keep: self,
@@ -80,6 +83,12 @@ impl super::super::Thread for Thread {
         let oldfs = FS::read_base().as_u64();
         let oldgs = GS::read_base().as_u64();
 
+        let block = if self.cssa == 1 || self.cssa == 2 {
+            self.block[self.cssa - 1].as_mut_ptr()
+        } else {
+            ptr::null_mut()
+        };
+
         // The `enclu` instruction consumes `rax`, `rbx` and `rcx`. However,
         // the vDSO function preserves `rbx` AND sets `rax` as the return
         // value. All other registers are passed to and from the enclave
@@ -99,7 +108,7 @@ impl super::super::Thread for Thread {
                 "pop  rbp",       // restore rbp
                 "pop  rbx",       // restore rbx
 
-                inout("rdi") self.block.as_mut_ptr() => _,
+                inout("rdi") block => _,
                 lateout("rsi") _,
                 lateout("rdx") _,
                 inout("rcx") how => _,
@@ -122,9 +131,16 @@ impl super::super::Thread for Thread {
 
         self.how = match run.function as usize {
             EENTER | ERESUME if run.vector == Vector::InvalidOpcode => EENTER,
-
-            #[cfg(feature = "gdb")]
-            EENTER | ERESUME if run.vector == Vector::Page => EENTER,
+            EENTER | ERESUME if run.vector == Vector::Page => {
+                trace!(
+                    "Vector::Page: address = {:>#016x}, error code = {:>#016b} cssa={}",
+                    run.exception_addr,
+                    run.exception_error_code,
+                    self.cssa
+                );
+
+                EENTER
+            }
 
             EEXIT if self.cssa > 0 => ERESUME,
             EEXIT if self.cssa == 0 => {
@@ -158,22 +174,15 @@ impl super::super::Thread for Thread {
             _ => unreachable!(),
         }
 
-        if self.cssa > 3 {
+        if self.cssa > 4 {
             error!("SGX CSSA overflow");
             bail!("SGX CSSA overflow");
         }
 
-        // If we have handled an InvalidOpcode error, evaluate the sallyport.
-        //
-        // Currently, we have no way to know if the sallyport contains a valid
-        // request by evaluating the sallyport directly. So we must presume
-        // that the sallyport is only valid when moving from CSSA 2 to CSSA 1.
-        //
-        // After the sallyport rework, we can test the sallyport itself and
-        // remove this logic.
-        if self.cssa > 0 {
+        // Handle some potential sallyport contents
+        if self.cssa == 1 || self.cssa == 2 {
             if let (EENTER, ERESUME) = (how, self.how) {
-                let block: Block = self.block.as_mut_slice().into();
+                let block: Block = self.block[self.cssa - 1].as_mut_slice().into();
                 for item in block {
                     match item {
                         Item::Gdbcall(_gdbcall, _data) => {