Merge pull request #1560 from SuHo-llrr/cfi-ext

Support Zicfiss (shadow stack access) with CFI extension v0.4.0

disasm/disasm.cc

@@ -266,6 +266,18 @@ struct : public arg_t {
   }
 } rvc_sp;
 
+struct : public arg_t {
+  std::string to_string(insn_t UNUSED insn) const {
+    return xpr_name[X_RA];
+  }
+} rvc_ra;
+
+struct : public arg_t {
+  std::string to_string(insn_t UNUSED insn) const {
+    return xpr_name[X_T0];
+  }
+} rvc_t0;
+
 struct : public arg_t {
   std::string to_string(insn_t insn) const {
     return std::to_string((int)insn.rvc_imm());
@@ -2184,6 +2196,13 @@ void disassembler_t::add_instructions(const isa_parser_t* isa)
   }
 
   if (isa->extension_enabled(EXT_ZIMOP)) {
+    #define DISASM_MOP_R(name, rs1, rd) \
+      add_insn(new disasm_insn_t(#name, match_##name | (rs1 << 15) | (rd << 7), \
+                                        0xFFFFFFFF, {&xrd, &xrs1}));
+
+    #define DISASM_MOP_RR(name, rs1, rd, rs2) \
+      add_insn(new disasm_insn_t(#name, match_##name | (rs1 << 15) | (rd << 7) | (rs2 << 20), \
+                                        0xFFFFFFFF, {&xrd, &xrs1, &xrs2}));
     DEFINE_R1TYPE(mop_r_0);
     DEFINE_R1TYPE(mop_r_1);
     DEFINE_R1TYPE(mop_r_2);
@@ -2212,7 +2231,15 @@ void disassembler_t::add_instructions(const isa_parser_t* isa)
     DEFINE_R1TYPE(mop_r_25);
     DEFINE_R1TYPE(mop_r_26);
     DEFINE_R1TYPE(mop_r_27);
-    DEFINE_R1TYPE(mop_r_28);
+    if (!isa->extension_enabled(EXT_ZICFISS)) {
+      DEFINE_R1TYPE(mop_r_28);
+    } else {
+      // Add code points of mop_r_28 not used by Zicfiss
+      for (unsigned rd_val = 0; rd_val <= 31; ++rd_val)
+        for (unsigned rs1_val = 0; rs1_val <= 31; ++rs1_val)
+          if ((rd_val != 0 && rs1_val !=0) || (rd_val == 0 && !(rs1_val == 1 || rs1_val == 5)))
+            DISASM_MOP_R(mop_r_28, rs1_val, rd_val);
+    }
     DEFINE_R1TYPE(mop_r_29);
     DEFINE_R1TYPE(mop_r_30);
     DEFINE_R1TYPE(mop_r_31);
@@ -2224,12 +2251,24 @@ void disassembler_t::add_instructions(const isa_parser_t* isa)
     DEFINE_RTYPE(mop_rr_5);
     DEFINE_RTYPE(mop_rr_6);
     DEFINE_RTYPE(mop_rr_7);
+    if (!isa->extension_enabled(EXT_ZICFISS)) {
+      DEFINE_RTYPE(mop_rr_7);
+    } else {
+      // Add code points of mop_rr_7 not used by Zicfiss
+      for (unsigned rd_val = 0; rd_val <= 31; ++rd_val)
+        for (unsigned rs1_val = 0; rs1_val <= 31; ++rs1_val)
+          for (unsigned rs2_val = 0; rs2_val <= 31; ++rs2_val)
+            if ((rs2_val != 1 && rs2_val != 5) || rd_val != 0 || rs1_val != 0)
+              DISASM_MOP_RR(mop_rr_7, rs1_val, rd_val, rs2_val);
+    }
   }
 
   if (isa->extension_enabled(EXT_ZCMOP)) {
-    DISASM_INSN("c.mop.1", c_mop_1, 0, {});
+    if (!isa->extension_enabled(EXT_ZICFISS))
+      DISASM_INSN("c.mop.1", c_mop_1, 0, {});
     DISASM_INSN("c.mop.3", c_mop_3, 0, {});
-    DISASM_INSN("c.mop.5", c_mop_5, 0, {});
+    if (!isa->extension_enabled(EXT_ZICFISS))
+      DISASM_INSN("c.mop.5", c_mop_5, 0, {});
     DISASM_INSN("c.mop.7", c_mop_7, 0, {});
     DISASM_INSN("c.mop.9", c_mop_9, 0, {});
     DISASM_INSN("c.mop.11", c_mop_11, 0, {});
@@ -2392,6 +2431,23 @@ void disassembler_t::add_instructions(const isa_parser_t* isa)
     DEFINE_XSTORE_BASE(sw_rl);
     DEFINE_XSTORE_BASE(sd_rl);
   }
+
+  if(isa->extension_enabled(EXT_ZICFISS)) {
+    DISASM_INSN("sspush", sspush_x1, 0, {&xrs2});
+    DISASM_INSN("sspush", sspush_x5, 0, {&xrs2});
+    DISASM_INSN("sspopchk", sspopchk_x1, 0, {&xrs1});
+    DISASM_INSN("sspopchk", sspopchk_x5, 0, {&xrs1});
+    DISASM_INSN("ssrdp", ssrdp, 0, {&xrd});
+    DEFINE_XAMO(ssamoswap_w);
+
+    if(isa->get_max_xlen() == 64)
+      DEFINE_XAMO(ssamoswap_d)
+
+    if (isa->extension_enabled(EXT_ZCA)) {
+      DISASM_INSN("c.sspush", c_sspush_x1, 0, {&rvc_ra});
+      DISASM_INSN("c.sspopchk", c_sspopchk_x5, 0, {&rvc_t0});
+    }
+  }
 }
 
 disassembler_t::disassembler_t(const isa_parser_t *isa)

disasm/isa_parser.cc

@@ -308,6 +308,8 @@ isa_parser_t::isa_parser_t(const char* str, const char *priv)
       extension_table[EXT_SSQOSID] = true;
     } else if (ext_str == "zicfilp") {
       extension_table[EXT_ZICFILP] = true;
+    } else if (ext_str == "zicfiss") {
+      extension_table[EXT_ZICFISS] = true;
     } else if (ext_str[0] == 'x') {
       extension_table['X'] = true;
       if (ext_str.size() == 1) {
@@ -397,6 +399,16 @@ isa_parser_t::isa_parser_t(const char* str, const char *priv)
       (extension_table[EXT_ZVKG] || extension_table[EXT_ZVKNED] || extension_table[EXT_ZVKSH])) {
     bad_isa_string(str, "'Zvkg', 'Zvkned', and 'Zvksh' extensions are incompatible with 'Zpn' extension in rv64");
   }
+
+  // When SSE is 0, Zicfiss behavior is defined by Zicmop
+  if (extension_table[EXT_ZICFISS] && !extension_table[EXT_ZIMOP]) {
+    bad_isa_string(str, "'Zicfiss' extension requires 'Zimop' extension");
+  }
+
+  if (extension_table[EXT_ZICFISS] && extension_table[EXT_ZCA] &&
+      !extension_table[EXT_ZCMOP]) {
+    bad_isa_string(str, "'Zicfiss' extension requires 'Zcmop' extension when `Zca` is supported");
+  }
 #ifdef WORDS_BIGENDIAN
   // Access to the vector registers as element groups is unimplemented on big-endian setups.
   if (extension_table[EXT_ZVKG] || extension_table[EXT_ZVKNHA] || extension_table[EXT_ZVKNHB] ||

riscv/csrs.cc

@@ -1757,3 +1757,13 @@ bool hvip_csr_t::unlogged_write(const reg_t val) noexcept {
   state->mip->write_with_mask(MIP_VSSIP, val); // hvip.VSSIP is an alias of mip.VSSIP
   return basic_csr_t::unlogged_write(val & (MIP_VSEIP | MIP_VSTIP));
 }
+
+ssp_csr_t::ssp_csr_t(processor_t* const proc, const reg_t addr, const reg_t mask, const reg_t init):
+  masked_csr_t(proc, addr, mask, init) {
+}
+
+void ssp_csr_t::verify_permissions(insn_t insn, bool write) const {
+  masked_csr_t::verify_permissions(insn, write);
+  DECLARE_XENVCFG_VARS(SSE);
+  require_envcfg(SSE);
+}

riscv/csrs.h

@@ -871,4 +871,11 @@ class hvip_csr_t : public basic_csr_t {
 };
 
 typedef std::shared_ptr<hvip_csr_t> hvip_csr_t_p;
+
+// ssp CSR provided by CFI Zicfiss extension
+class ssp_csr_t final : public masked_csr_t {
+ public:
+  ssp_csr_t(processor_t* const proc, const reg_t addr, const reg_t mask, const reg_t init);
+  virtual void verify_permissions(insn_t insn, bool write) const override;
+};
 #endif

riscv/decode.h

@@ -26,6 +26,7 @@ const int NCSR = 4096;
 
 #define X_RA 1
 #define X_SP 2
+#define X_T0 5
 #define X_S0 8
 #define X_A0 10
 #define X_A1 11

riscv/encoding.h

@@ -348,6 +348,7 @@
 
 /* software check exception xtval codes */
 #define LANDING_PAD_FAULT 2
+#define SHADOW_STACK_FAULT 3
 
 #ifdef __riscv
 

riscv/insns/c_lui.h

@@ -5,7 +5,13 @@ if (insn.rvc_rd() == 2) { // c.addi16sp
 } else if (insn.rvc_imm() != 0) { // c.lui
   WRITE_RD(insn.rvc_imm() << 12);
 } else if ((insn.rvc_rd() & 0x11) == 1) { // c.mop.N
-  #include "c_mop_N.h"
+  if (insn.rvc_rd() == 5 && p->extension_enabled(EXT_ZICFISS)) {
+    #include "c_sspopchk_x5.h"
+  } else if (insn.rvc_rd() == 1 && p->extension_enabled(EXT_ZICFISS)) {
+    #include "c_sspush_x1.h"
+  } else {
+    #include "c_mop_N.h"
+  }
 } else {
   require(false);
 }

riscv/insns/c_sspopchk_x5.h

@@ -0,0 +1,5 @@
+#include "zicfiss.h"
+
+if (xSSE()) {
+  POP_VALUE_FROM_SS_AND_CHECK(READ_REG(X_T0));
+}

riscv/insns/c_sspush_x1.h

@@ -0,0 +1,5 @@
+#include "zicfiss.h"
+
+if (xSSE()) {
+  PUSH_VALUE_TO_SS(RA);
+}

riscv/insns/ssamoswap_d.h

@@ -0,0 +1,7 @@
+require_extension(EXT_ZICFISS);
+require_extension('A');
+require_rv64;
+
+DECLARE_XENVCFG_VARS(SSE);
+require_envcfg(SSE);
+WRITE_RD(MMU.ssamoswap<uint64_t>(RS1, RS2));

riscv/insns/ssamoswap_w.h

@@ -0,0 +1,7 @@
+require_extension(EXT_ZICFISS);
+require_extension('A');
+
+DECLARE_XENVCFG_VARS(SSE);
+require_envcfg(SSE);
+WRITE_RD(sext32(MMU.ssamoswap<uint32_t>(RS1, RS2)));
+

riscv/insns/sspopchk_x1.h

@@ -0,0 +1,7 @@
+#include "zicfiss.h"
+
+if (xSSE()) {
+  POP_VALUE_FROM_SS_AND_CHECK(RS1);
+} else {
+  #include "mop_r_N.h"
+}

riscv/insns/sspopchk_x5.h

@@ -0,0 +1 @@
+#include "sspopchk_x1.h"

riscv/insns/sspush_x1.h

@@ -0,0 +1,7 @@
+#include "zicfiss.h"
+
+if (xSSE()) {
+  PUSH_VALUE_TO_SS(RS2);
+} else {
+  #include "mop_rr_N.h"
+}

riscv/insns/sspush_x5.h

@@ -0,0 +1 @@
+#include "sspush_x1.h"

riscv/insns/ssrdp.h

@@ -0,0 +1,7 @@
+#include "zicfiss.h"
+
+if (xSSE()) {
+  WRITE_RD(STATE.ssp->read());
+} else {
+  #include "mop_r_N.h"
+}

riscv/isa_parser.h

@@ -88,6 +88,7 @@ typedef enum {
   EXT_ZALASR,
   EXT_SSQOSID,
   EXT_ZICFILP,
+  EXT_ZICFISS,
   NUM_ISA_EXTENSIONS
 } isa_extension_t;
 

riscv/mmu.cc

@@ -63,7 +63,7 @@ reg_t mmu_t::translate(mem_access_info_t access_info, reg_t len)
   reg_t mode = (reg_t) access_info.effective_priv;
 
   reg_t paddr = walk(access_info) | (addr & (PGSIZE-1));
-  if (!pmp_ok(paddr, len, type, mode))
+  if (!pmp_ok(paddr, len, access_info.flags.ss_access ? STORE : type, mode))
     throw_access_exception(virt, addr, type);
   return paddr;
 }
@@ -491,6 +491,15 @@ reg_t mmu_t::walk(mem_access_info_t access_info)
   reg_t page_mask = (reg_t(1) << PGSHIFT) - 1;
   reg_t satp = proc->get_state()->satp->readvirt(virt);
   vm_info vm = decode_vm_info(proc->get_const_xlen(), false, mode, satp);
+
+  bool ss_access = access_info.flags.ss_access;
+
+  if (ss_access) {
+    if (vm.levels == 0)
+      trap_store_access_fault(virt, addr, 0, 0);
+    type = STORE;
+  }
+
   if (vm.levels == 0)
     return s2xlate(addr, addr & ((reg_t(2) << (proc->xlen-1))-1), type, type, virt, hlvx, false) & ~page_mask; // zero-extend from xlen
 
@@ -516,6 +525,7 @@ reg_t mmu_t::walk(mem_access_info_t access_info)
     reg_t ppn = (pte & ~reg_t(PTE_ATTR)) >> PTE_PPN_SHIFT;
     bool pbmte = virt ? (proc->get_state()->henvcfg->read() & HENVCFG_PBMTE) : (proc->get_state()->menvcfg->read() & MENVCFG_PBMTE);
     bool hade = virt ? (proc->get_state()->henvcfg->read() & HENVCFG_ADUE) : (proc->get_state()->menvcfg->read() & MENVCFG_ADUE);
+    bool sse = virt ? (proc->get_state()->henvcfg->read() & HENVCFG_SSE) : (proc->get_state()->menvcfg->read() & MENVCFG_SSE);
 
     if (pte & PTE_RSVD) {
       break;
@@ -531,11 +541,24 @@ reg_t mmu_t::walk(mem_access_info_t access_info)
       base = ppn << PGSHIFT;
     } else if ((pte & PTE_U) ? s_mode && (type == FETCH || !sum) : !s_mode) {
       break;
-    } else if (!(pte & PTE_V) || (!(pte & PTE_R) && (pte & PTE_W))) {
+    } else if (!(pte & PTE_V) ||
+              (!(pte & PTE_R) && (pte & PTE_W) && ((!sse && !(pte & PTE_X)) || (pte & PTE_X)))) {
+      // invalid
+      // not shadow stack access xwr=110 or xwr=010 page cause page fault
+      // shadow stack access with PTE_X moved to following check
       break;
+    } else if ((!(pte & PTE_R) && (pte & PTE_W) && !(pte & PTE_X)) && (type == STORE && !ss_access)) {
+      // not shadow stack store and  xwr = 010 cause access-fault
+      throw trap_store_access_fault(virt, addr, 0, 0);
+    } else if ((!(pte & PTE_R) && (pte & PTE_W) && !(pte & PTE_X)) && type == FETCH) {
+      // fetch from shadow stack pages cause instruction access-fault
+      throw trap_instruction_access_fault(virt, addr, 0, 0);
+    } else if ((((pte & PTE_R) && (pte & PTE_W)) || (pte & PTE_X)) && ss_access) {
+      // shadow stack access cause store access fault if xwr!=010 and xwr!=001
+      throw trap_store_access_fault(virt, addr, 0, 0);
     } else if (type == FETCH || hlvx ? !(pte & PTE_X) :
                type == LOAD          ? !(pte & PTE_R) && !(mxr && (pte & PTE_X)) :
-                                       !((pte & PTE_R) && (pte & PTE_W))) {
+                                       !(pte & PTE_W)) {
       break;
     } else if ((ppn & ((reg_t(1) << ptshift) - 1)) != 0) {
       break;

riscv/mmu.h

@@ -42,9 +42,10 @@ struct xlate_flags_t {
   const bool forced_virt : 1 {false};
   const bool hlvx : 1 {false};
   const bool lr : 1 {false};
+  const bool ss_access : 1 {false};
 
   bool is_special_access() const {
-    return forced_virt || hlvx || lr;
+    return forced_virt || hlvx || lr || ss_access;
   }
 };
 
@@ -127,6 +128,14 @@ class mmu_t
     return load<T>(addr, {.forced_virt=true, .hlvx=true});
   }
 
+  // shadow stack load
+  template<typename T>
+  T ss_load(reg_t addr) {
+    if ((addr & (sizeof(T) - 1)) != 0)
+      throw trap_store_access_fault((proc) ? proc->state.v : false, addr, 0, 0);
+    return load<T>(addr, {.forced_virt=false, .hlvx=false, .lr=false, .ss_access=true});
+  }
+
   template<typename T>
   void ALWAYS_INLINE store(reg_t addr, T val, xlate_flags_t xlate_flags = {}) {
     reg_t vpn = addr >> PGSHIFT;
@@ -149,6 +158,14 @@ class mmu_t
     store(addr, val, {.forced_virt=true});
   }
 
+  // shadow stack store
+  template<typename T>
+  void ss_store(reg_t addr, T val) {
+    if ((addr & (sizeof(T) - 1)) != 0)
+      throw trap_store_access_fault((proc) ? proc->state.v : false, addr, 0, 0);
+    store<T>(addr, val, {.forced_virt=false, .hlvx=false, .lr=false, .ss_access=true});
+  }
+
   // AMO/Zicbom faults should be reported as store faults
   #define convert_load_traps_to_store_traps(BODY) \
     try { \
@@ -175,6 +192,19 @@ class mmu_t
     })
   }
 
+  // for shadow stack amoswap
+  template<typename T>
+  T ssamoswap(reg_t addr, reg_t value) {
+      bool forced_virt = false;
+      bool hlvx = false;
+      bool lr = false;
+      bool ss_access = true;
+      store_slow_path(addr, sizeof(T), nullptr, {forced_virt, hlvx, lr, ss_access}, false, true);
+      auto data = load<T>(addr, {forced_virt, hlvx, lr, ss_access});
+      store<T>(addr, value, {forced_virt, hlvx, lr, ss_access});
+      return data;
+  }
+
   template<typename T>
   T amo_compare_and_swap(reg_t addr, T comp, T swap) {
     convert_load_traps_to_store_traps({

riscv/overlap_list.h

@@ -22,3 +22,10 @@ DECLARE_OVERLAP_INSN(rstsa32, EXT_ZPN)
 DECLARE_OVERLAP_INSN(srli32_u, EXT_ZPN)
 DECLARE_OVERLAP_INSN(umax32, EXT_ZPN)
 DECLARE_OVERLAP_INSN(lpad, EXT_ZICFILP)
+DECLARE_OVERLAP_INSN(mop_r_28, EXT_ZIMOP)
+DECLARE_OVERLAP_INSN(mop_r_N, EXT_ZIMOP)
+DECLARE_OVERLAP_INSN(mop_rr_7, EXT_ZIMOP)
+DECLARE_OVERLAP_INSN(mop_rr_N, EXT_ZIMOP)
+DECLARE_OVERLAP_INSN(c_sspush_x1, EXT_ZICFISS)
+DECLARE_OVERLAP_INSN(c_sspopchk_x5, EXT_ZICFISS)
+DECLARE_OVERLAP_INSN(c_mop_N, EXT_ZCMOP)