runtime/src/enclave_rpc: Verify RPC quotes with key manager quote policy

.changelog/5092.bugfix.md

@@ -0,0 +1,5 @@
+go/runtime/registry: Fix watching policy updates
+
+When multiple key managers were running, the last known status of the
+runtime's key manager was overwritten with each status update. On runtime
+(re)starts, this resulted in the wrong policy being set.

.changelog/5092.feature.md

@@ -0,0 +1 @@
+runtime/src/enclave_rpc: Verify RPC quotes with key manager quote policy

go/common/namespace.go

@@ -105,7 +105,10 @@ func (n *Namespace) UnmarshalBase64(text []byte) error {
 
 // Equal compares vs another namespace for equality.
 func (n *Namespace) Equal(cmp *Namespace) bool {
-	if cmp == nil {
+	if n == cmp {
+		return true
+	}
+	if n == nil || cmp == nil {
 		return false
 	}
 	return bytes.Equal(n[:], cmp[:])

go/registry/api/runtime.go

@@ -648,6 +648,12 @@ type VersionInfo struct {
 
 // Equal compares vs another VersionInfo for equality.
 func (vi *VersionInfo) Equal(cmp *VersionInfo) bool {
+	if vi == cmp {
+		return true
+	}
+	if vi == nil || cmp == nil {
+		return false
+	}
 	if vi.Version.ToU64() != cmp.Version.ToU64() {
 		return false
 	}

go/runtime/host/multi/multi.go

@@ -192,7 +192,7 @@ func (agg *Aggregate) SetVersion(ctx context.Context, version version.Version) e
 			return nil
 		}
 
-		// Otherwise tear it dow.
+		// Otherwise tear it down.
 		agg.stopActiveLocked()
 	}
 

go/runtime/host/protocol/types.go

@@ -67,34 +67,36 @@ type Body struct {
 	Error *Error `json:",omitempty"`
 
 	// Runtime interface.
-	RuntimeInfoRequest                    *RuntimeInfoRequest                    `json:",omitempty"`
-	RuntimeInfoResponse                   *RuntimeInfoResponse                   `json:",omitempty"`
-	RuntimePingRequest                    *Empty                                 `json:",omitempty"`
-	RuntimeShutdownRequest                *Empty                                 `json:",omitempty"`
-	RuntimeCapabilityTEERakInitRequest    *RuntimeCapabilityTEERakInitRequest    `json:",omitempty"`
-	RuntimeCapabilityTEERakInitResponse   *Empty                                 `json:",omitempty"`
-	RuntimeCapabilityTEERakReportRequest  *Empty                                 `json:",omitempty"`
-	RuntimeCapabilityTEERakReportResponse *RuntimeCapabilityTEERakReportResponse `json:",omitempty"`
-	RuntimeCapabilityTEERakAvrRequest     *RuntimeCapabilityTEERakAvrRequest     `json:",omitempty"`
-	RuntimeCapabilityTEERakAvrResponse    *Empty                                 `json:",omitempty"`
-	RuntimeCapabilityTEERakQuoteRequest   *RuntimeCapabilityTEERakQuoteRequest   `json:",omitempty"`
-	RuntimeCapabilityTEERakQuoteResponse  *RuntimeCapabilityTEERakQuoteResponse  `json:",omitempty"`
-	RuntimeRPCCallRequest                 *RuntimeRPCCallRequest                 `json:",omitempty"`
-	RuntimeRPCCallResponse                *RuntimeRPCCallResponse                `json:",omitempty"`
-	RuntimeLocalRPCCallRequest            *RuntimeLocalRPCCallRequest            `json:",omitempty"`
-	RuntimeLocalRPCCallResponse           *RuntimeLocalRPCCallResponse           `json:",omitempty"`
-	RuntimeCheckTxBatchRequest            *RuntimeCheckTxBatchRequest            `json:",omitempty"`
-	RuntimeCheckTxBatchResponse           *RuntimeCheckTxBatchResponse           `json:",omitempty"`
-	RuntimeExecuteTxBatchRequest          *RuntimeExecuteTxBatchRequest          `json:",omitempty"`
-	RuntimeExecuteTxBatchResponse         *RuntimeExecuteTxBatchResponse         `json:",omitempty"`
-	RuntimeAbortRequest                   *Empty                                 `json:",omitempty"`
-	RuntimeAbortResponse                  *Empty                                 `json:",omitempty"`
-	RuntimeKeyManagerPolicyUpdateRequest  *RuntimeKeyManagerPolicyUpdateRequest  `json:",omitempty"`
-	RuntimeKeyManagerPolicyUpdateResponse *Empty                                 `json:",omitempty"`
-	RuntimeQueryRequest                   *RuntimeQueryRequest                   `json:",omitempty"`
-	RuntimeQueryResponse                  *RuntimeQueryResponse                  `json:",omitempty"`
-	RuntimeConsensusSyncRequest           *RuntimeConsensusSyncRequest           `json:",omitempty"`
-	RuntimeConsensusSyncResponse          *Empty                                 `json:",omitempty"`
+	RuntimeInfoRequest                         *RuntimeInfoRequest                        `json:",omitempty"`
+	RuntimeInfoResponse                        *RuntimeInfoResponse                       `json:",omitempty"`
+	RuntimePingRequest                         *Empty                                     `json:",omitempty"`
+	RuntimeShutdownRequest                     *Empty                                     `json:",omitempty"`
+	RuntimeCapabilityTEERakInitRequest         *RuntimeCapabilityTEERakInitRequest        `json:",omitempty"`
+	RuntimeCapabilityTEERakInitResponse        *Empty                                     `json:",omitempty"`
+	RuntimeCapabilityTEERakReportRequest       *Empty                                     `json:",omitempty"`
+	RuntimeCapabilityTEERakReportResponse      *RuntimeCapabilityTEERakReportResponse     `json:",omitempty"`
+	RuntimeCapabilityTEERakAvrRequest          *RuntimeCapabilityTEERakAvrRequest         `json:",omitempty"`
+	RuntimeCapabilityTEERakAvrResponse         *Empty                                     `json:",omitempty"`
+	RuntimeCapabilityTEERakQuoteRequest        *RuntimeCapabilityTEERakQuoteRequest       `json:",omitempty"`
+	RuntimeCapabilityTEERakQuoteResponse       *RuntimeCapabilityTEERakQuoteResponse      `json:",omitempty"`
+	RuntimeRPCCallRequest                      *RuntimeRPCCallRequest                     `json:",omitempty"`
+	RuntimeRPCCallResponse                     *RuntimeRPCCallResponse                    `json:",omitempty"`
+	RuntimeLocalRPCCallRequest                 *RuntimeLocalRPCCallRequest                `json:",omitempty"`
+	RuntimeLocalRPCCallResponse                *RuntimeLocalRPCCallResponse               `json:",omitempty"`
+	RuntimeCheckTxBatchRequest                 *RuntimeCheckTxBatchRequest                `json:",omitempty"`
+	RuntimeCheckTxBatchResponse                *RuntimeCheckTxBatchResponse               `json:",omitempty"`
+	RuntimeExecuteTxBatchRequest               *RuntimeExecuteTxBatchRequest              `json:",omitempty"`
+	RuntimeExecuteTxBatchResponse              *RuntimeExecuteTxBatchResponse             `json:",omitempty"`
+	RuntimeAbortRequest                        *Empty                                     `json:",omitempty"`
+	RuntimeAbortResponse                       *Empty                                     `json:",omitempty"`
+	RuntimeKeyManagerPolicyUpdateRequest       *RuntimeKeyManagerPolicyUpdateRequest      `json:",omitempty"`
+	RuntimeKeyManagerPolicyUpdateResponse      *Empty                                     `json:",omitempty"`
+	RuntimeKeyManagerQuotePolicyUpdateRequest  *RuntimeKeyManagerQuotePolicyUpdateRequest `json:",omitempty"`
+	RuntimeKeyManagerQuotePolicyUpdateResponse *Empty                                     `json:",omitempty"`
+	RuntimeQueryRequest                        *RuntimeQueryRequest                       `json:",omitempty"`
+	RuntimeQueryResponse                       *RuntimeQueryResponse                      `json:",omitempty"`
+	RuntimeConsensusSyncRequest                *RuntimeConsensusSyncRequest               `json:",omitempty"`
+	RuntimeConsensusSyncResponse               *Empty                                     `json:",omitempty"`
 
 	// Host interface.
 	HostRPCCallRequest               *HostRPCCallRequest               `json:",omitempty"`
@@ -395,12 +397,17 @@ type RuntimeExecuteTxBatchResponse struct {
 	Deprecated1 cbor.RawMessage `json:"batch_weight_limits,omitempty"`
 }
 
-// RuntimeKeyManagerPolicyUpdateRequest is a runtime key manager policy request
-// message body.
+// RuntimeKeyManagerPolicyUpdateRequest is a runtime key manager policy request message body.
 type RuntimeKeyManagerPolicyUpdateRequest struct {
 	SignedPolicyRaw []byte `json:"signed_policy_raw"`
 }
 
+// RuntimeKeyManagerQuotePolicyUpdateRequest is a runtime key manager quote policy request
+// message body.
+type RuntimeKeyManagerQuotePolicyUpdateRequest struct {
+	Policy quote.Policy `json:"policy"`
+}
+
 // RuntimeQueryRequest is a runtime query request message body.
 type RuntimeQueryRequest struct {
 	// ConsensusBlock is the consensus light block at the last finalized round

go/runtime/registry/host.go

@@ -9,9 +9,12 @@ import (
 
 	"github.com/eapache/channels"
 
+	"github.com/oasisprotocol/oasis-core/go/common"
 	"github.com/oasisprotocol/oasis-core/go/common/cbor"
 	"github.com/oasisprotocol/oasis-core/go/common/identity"
 	"github.com/oasisprotocol/oasis-core/go/common/logging"
+	"github.com/oasisprotocol/oasis-core/go/common/node"
+	"github.com/oasisprotocol/oasis-core/go/common/sgx/quote"
 	"github.com/oasisprotocol/oasis-core/go/common/version"
 	consensus "github.com/oasisprotocol/oasis-core/go/consensus/api"
 	consensusResults "github.com/oasisprotocol/oasis-core/go/consensus/api/transaction/results"
@@ -444,12 +447,78 @@ func (n *runtimeHostNotifier) watchPolicyUpdates() {
 	}
 	defer dscSub.Close()
 
-	// Subscribe to key manager status updates.
+	var (
+		kmRtID *common.Namespace
+		done   bool
+	)
+
+	for !done {
+		done = func() bool {
+			// Start watching key manager policy updates.
+			var wg sync.WaitGroup
+			defer wg.Wait()
+
+			ctx, cancel := context.WithCancel(n.ctx)
+			defer cancel()
+
+			wg.Add(1)
+			go func(kmRtID *common.Namespace) {
+				defer wg.Done()
+				n.watchKmPolicyUpdates(ctx, kmRtID)
+			}(kmRtID)
+
+			// Restart the updater if the runtime changes the key manager. This should happen
+			// at most once as runtimes are not allowed to change the manager once set.
+			for {
+				select {
+				case <-n.ctx.Done():
+					n.logger.Debug("context canceled")
+					return true
+				case <-n.stopCh:
+					n.logger.Debug("termination requested")
+					return true
+				case rtDsc := <-dscCh:
+					n.logger.Debug("got registry descriptor update")
+
+					if rtDsc.Kind != registry.KindCompute {
+						return true
+					}
+
+					if kmRtID.Equal(rtDsc.KeyManager) {
+						break
+					}
+
+					kmRtID = rtDsc.KeyManager
+					return false
+				}
+			}
+		}()
+	}
+}
+
+func (n *runtimeHostNotifier) watchKmPolicyUpdates(ctx context.Context, kmRtID *common.Namespace) {
+	// No need to watch anything if key manager is not set.
+	if kmRtID == nil {
+		return
+	}
+
+	n.logger.Debug("watching key manager policy updates", "keymanager", kmRtID)
+
+	// Subscribe to key manager status updates (policy might change).
 	stCh, stSub := n.consensus.KeyManager().WatchStatuses()
 	defer stSub.Close()
-	n.logger.Debug("watching policy updates")
 
-	// Subscribe to runtime host events.
+	// Subscribe to epoch transitions (quote policy might change).
+	epoCh, sub, err := n.consensus.Beacon().WatchEpochs(ctx)
+	if err != nil {
+		n.logger.Error("failed to watch epochs",
+			"err", err,
+		)
+		return
+	}
+	defer sub.Close()
+
+	// Subscribe to runtime host events (policies will be lost on restarts).
 	evCh, evSub, err := n.host.WatchEvents(n.ctx)
 	if err != nil {
 		n.logger.Error("failed to subscribe to runtime host events",
@@ -460,74 +529,114 @@ func (n *runtimeHostNotifier) watchPolicyUpdates() {
 	defer evSub.Close()
 
 	var (
-		rtDsc *registry.Runtime
-		st    *keymanager.Status
+		st *keymanager.Status
+		sc *node.SGXConstraints
+		vi *registry.VersionInfo
 	)
+
 	for {
 		select {
-		case <-n.ctx.Done():
-			n.logger.Debug("context canceled")
+		case <-ctx.Done():
 			return
-		case <-n.stopCh:
-			n.logger.Debug("termination requested")
-			return
-		case rtDsc = <-dscCh:
-			n.logger.Debug("got registry descriptor update")
-
-			// Ignore updates if key manager is not needed.
-			if rtDsc.KeyManager == nil {
-				n.logger.Debug("no key manager needed for this runtime")
+		case newSt := <-stCh:
+			// Ignore status updates for a different key manager.
+			if !newSt.ID.Equal(kmRtID) {
 				continue
 			}
+			st = newSt
 
-			var err error
-			st, err = n.consensus.KeyManager().GetStatus(n.ctx, &registry.NamespaceQuery{
+			n.updateKeyManagerPolicy(ctx, st.Policy)
+		case epoch := <-epoCh:
+			// Check if the key manager was redeployed, as that is when a new quote policy might
+			// take effect.
+			dsc, err := n.consensus.Registry().GetRuntime(ctx, &registry.GetRuntimeQuery{
 				Height: consensus.HeightLatest,
-				ID:     *rtDsc.KeyManager,
+				ID:     *kmRtID,
 			})
 			if err != nil {
-				n.logger.Warn("failed to fetch key manager status",
+				n.logger.Error("failed to query key manager runtime descriptor",
 					"err", err,
 				)
 				continue
 			}
-		case st = <-stCh:
-			// Ignore status updates if key manager is not yet known (is nil)
-			// or if the status update is for a different key manager.
-			if rtDsc == nil || !st.ID.Equal(rtDsc.KeyManager) {
+
+			// Quote polices can only be set on SGX hardwares.
+			if dsc.TEEHardware != node.TEEHardwareIntelSGX {
+				continue
+			}
+
+			// No need to update the policy if the key manager is sill running the same version.
+			newVi := dsc.ActiveDeployment(epoch)
+			if newVi.Equal(vi) {
 				continue
 			}
+			vi = newVi
+
+			// Parse SGX constraints.
+			var newSc node.SGXConstraints
+			if err := cbor.Unmarshal(vi.TEE, &newSc); err != nil {
+				n.logger.Error("malformed SGX constraints",
+					"err", err,
+				)
+				continue
+			}
+			sc = &newSc
+
+			n.updateKeyManagerQuotePolicy(ctx, sc.Policy)
 		case ev := <-evCh:
-			// Runtime host changes, make sure to update the policy if runtime is restarted.
+			// Runtime host changes, make sure to update the policies if runtime is restarted.
 			if ev.Started == nil && ev.Updated == nil {
 				continue
 			}
+			// Make sure that we actually have policies.
+			if st != nil {
+				n.updateKeyManagerPolicy(ctx, st.Policy)
+			}
+			if sc != nil {
+				n.updateKeyManagerQuotePolicy(ctx, sc.Policy)
+			}
 		}
+	}
+}
 
-		// Make sure that we actually have a policy.
-		if st == nil {
-			continue
-		}
+func (n *runtimeHostNotifier) updateKeyManagerPolicy(ctx context.Context, policy *keymanager.SignedPolicySGX) {
+	n.logger.Debug("got key manager policy update", "policy", policy)
 
-		// Update key manager policy.
-		n.logger.Debug("got policy update", "status", st)
+	raw := cbor.Marshal(policy)
+	req := &protocol.Body{RuntimeKeyManagerPolicyUpdateRequest: &protocol.RuntimeKeyManagerPolicyUpdateRequest{
+		SignedPolicyRaw: raw,
+	}}
 
-		raw := cbor.Marshal(st.Policy)
-		req := &protocol.Body{RuntimeKeyManagerPolicyUpdateRequest: &protocol.RuntimeKeyManagerPolicyUpdateRequest{
-			SignedPolicyRaw: raw,
-		}}
+	ctx, cancel := context.WithTimeout(ctx, notifyTimeout)
+	defer cancel()
 
-		ctx, cancel := context.WithTimeout(n.ctx, notifyTimeout)
-		response, err := n.host.Call(ctx, req)
-		cancel()
-		if err != nil {
-			n.logger.Error("failed dispatching key manager policy update to runtime",
-				"err", err,
-			)
-			continue
-		}
-		n.logger.Debug("key manager policy updated dispatched", "response", response)
+	if _, err := n.host.Call(ctx, req); err != nil {
+		n.logger.Error("failed dispatching key manager policy update to runtime",
+			"err", err,
+		)
+		return
+	}
+
+	n.logger.Debug("key manager policy update dispatched")
+}
+
+func (n *runtimeHostNotifier) updateKeyManagerQuotePolicy(ctx context.Context, policy *quote.Policy) {
+	n.logger.Debug("got key manager quote policy update", "policy", policy)
+
+	req := &protocol.Body{RuntimeKeyManagerQuotePolicyUpdateRequest: &protocol.RuntimeKeyManagerQuotePolicyUpdateRequest{
+		Policy: *policy,
+	}}
+
+	ctx, cancel := context.WithTimeout(ctx, notifyTimeout)
+	defer cancel()
+
+	if _, err := n.host.Call(ctx, req); err != nil {
+		n.logger.Error("failed dispatching key manager quote policy update to runtime",
+			"err", err,
+		)
+		return
 	}
+	n.logger.Debug("key manager quote policy update dispatched")
 }
 
 func (n *runtimeHostNotifier) watchConsensusLightBlocks() {

keymanager/src/api/errors.rs

@@ -27,8 +27,6 @@ pub enum KeyManagerError {
     PolicyInvalid(#[from] anyhow::Error),
     #[error("policy has insufficient signatures")]
     PolicyInsufficientSignatures,
-    #[error("policy hasn't been published")]
-    PolicyNotPublished,
     #[error(transparent)]
     Other(anyhow::Error),
 }