Skip to content

Commit

Permalink
go/runtime/registry: Fix key manager (quote) policy updates
Browse files Browse the repository at this point in the history
When a key manager (quote) policy update fails, the host should retry the
update until the policy is updated. For example, when using Tendermint as
a backend service, the first update will always fail because the consensus
verifier sees new blocks with a one-block delay.
  • Loading branch information
peternose committed Dec 26, 2022
1 parent 2530a6d commit d438583
Show file tree
Hide file tree
Showing 6 changed files with 89 additions and 81 deletions.
6 changes: 6 additions & 0 deletions .changelog/5111.bugfix.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
go/runtime/registry: Fix key manager (quote) policy updates

When a key manager (quote) policy update fails, the host should retry the
update until the policy is updated. For example, when using Tendermint as
a backend service, the first update will always fail because the consensus
verifier sees new blocks with a one-block delay.
88 changes: 60 additions & 28 deletions go/runtime/registry/host.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,13 @@ import (
"github.com/oasisprotocol/oasis-core/go/storage/mkvs/syncer"
)

// notifyTimeout is the maximum time to wait for a notification to be processed by the runtime.
const notifyTimeout = 10 * time.Second
const (
// notifyTimeout is the maximum time to wait for a notification to be processed by the runtime.
notifyTimeout = 10 * time.Second

// retryInterval is the time interval used between failed key manager updates.
retryInterval = time.Second
)

// RuntimeHostNode provides methods for nodes that need to host runtimes.
type RuntimeHostNode struct {
Expand Down Expand Up @@ -538,22 +543,49 @@ func (n *runtimeHostNotifier) watchKmPolicyUpdates(ctx context.Context, kmRtID *
}
defer evSub.Close()

// Fetch runtime info so that we know which features the runtime supports.
rtInfo, err := n.host.GetInfo(ctx)
if err != nil {
n.logger.Error("failed to fetch runtime info",
"err", err,
)
return
}
retryTicker := time.NewTicker(retryInterval)
defer retryTicker.Stop()

var (
policyUpdated = true
quotePolicyUpdated = true
runtimeInfoUpdated = false
)

var (
st *keymanager.Status
sc *node.SGXConstraints
vi *registry.VersionInfo
ri *protocol.RuntimeInfoResponse
)

for {
// Fetch runtime info so that we know which features the current runtime version supports.
if !runtimeInfoUpdated {
if ri, err = n.host.GetInfo(ctx); err != nil {
n.logger.Error("failed to fetch runtime info",
"err", err,
)
return
}
runtimeInfoUpdated = true
}

// Make sure that we actually have a new policy.
if !policyUpdated && st != nil && st.Policy != nil {
if err = n.updateKeyManagerPolicy(ctx, st.Policy); err == nil {
policyUpdated = true
}
}

// Make sure that we actually have a new quote policy and that the current runtime version
// supports quote policy updates.
if !quotePolicyUpdated && sc != nil && sc.Policy != nil && ri.Features.KeyManagerQuotePolicyUpdates {
if err = n.updateKeyManagerQuotePolicy(ctx, sc.Policy); err == nil {
quotePolicyUpdated = true
}
}

select {
case <-ctx.Done():
return
Expand All @@ -564,13 +596,8 @@ func (n *runtimeHostNotifier) watchKmPolicyUpdates(ctx context.Context, kmRtID *
}
st = newSt

n.updateKeyManagerPolicy(ctx, st.Policy)
policyUpdated = false
case epoch := <-epoCh:
// Skip quote policy updates if the runtime doesn't support them.
if !rtInfo.Features.KeyManagerQuotePolicyUpdates {
continue
}

// Check if the key manager was redeployed, as that is when a new quote policy might
// take effect.
dsc, err := n.consensus.Registry().GetRuntime(ctx, &registry.GetRuntimeQuery{
Expand Down Expand Up @@ -606,24 +633,27 @@ func (n *runtimeHostNotifier) watchKmPolicyUpdates(ctx context.Context, kmRtID *
}
sc = &newSc

n.updateKeyManagerQuotePolicy(ctx, sc.Policy)
quotePolicyUpdated = false
case ev := <-evCh:
// Runtime host changes, make sure to update the policies if runtime is restarted.
if ev.Started == nil && ev.Updated == nil {
continue
}
// Make sure that we actually have policies.
if st != nil {
n.updateKeyManagerPolicy(ctx, st.Policy)
}
if sc != nil {
n.updateKeyManagerQuotePolicy(ctx, sc.Policy)
}

policyUpdated = false
quotePolicyUpdated = false
runtimeInfoUpdated = false
case <-retryTicker.C:
// Retry updates if some of them failed. When using Tendermint as a backend service
// the host will see the new state one block before the consensus verifier as the former
// sees the block H after it is executed while the latter needs to trust the block H
// first by verifying the signatures which are only available after the block H+1
// finalizes.
}
}
}

func (n *runtimeHostNotifier) updateKeyManagerPolicy(ctx context.Context, policy *keymanager.SignedPolicySGX) {
func (n *runtimeHostNotifier) updateKeyManagerPolicy(ctx context.Context, policy *keymanager.SignedPolicySGX) error {
n.logger.Debug("got key manager policy update", "policy", policy)

raw := cbor.Marshal(policy)
Expand All @@ -638,13 +668,14 @@ func (n *runtimeHostNotifier) updateKeyManagerPolicy(ctx context.Context, policy
n.logger.Error("failed dispatching key manager policy update to runtime",
"err", err,
)
return
return err
}

n.logger.Debug("key manager policy update dispatched")
return nil
}

func (n *runtimeHostNotifier) updateKeyManagerQuotePolicy(ctx context.Context, policy *quote.Policy) {
func (n *runtimeHostNotifier) updateKeyManagerQuotePolicy(ctx context.Context, policy *quote.Policy) error {
n.logger.Debug("got key manager quote policy update", "policy", policy)

req := &protocol.Body{RuntimeKeyManagerQuotePolicyUpdateRequest: &protocol.RuntimeKeyManagerQuotePolicyUpdateRequest{
Expand All @@ -658,9 +689,10 @@ func (n *runtimeHostNotifier) updateKeyManagerQuotePolicy(ctx context.Context, p
n.logger.Error("failed dispatching key manager quote policy update to runtime",
"err", err,
)
return
return err
}
n.logger.Debug("key manager quote policy update dispatched")
return nil
}

func (n *runtimeHostNotifier) watchConsensusLightBlocks() {
Expand Down
1 change: 0 additions & 1 deletion keymanager/src/policy/cached.rs
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,6 @@ impl Policy {
ioctx,
untrusted_policy,
key_manager,
false,
)?;
let new_policy = CachedPolicy::parse(published_policy, policy_raw)?;

Expand Down
1 change: 0 additions & 1 deletion runtime/src/attestation.rs
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,6 @@ impl Handler {
ctx,
&self.runtime_id,
version,
true,
)?;

self.rak.set_quote_policy(policy)?;
Expand Down
25 changes: 10 additions & 15 deletions runtime/src/dispatcher.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1022,14 +1022,12 @@ impl Dispatcher {
let runtime_id = state.protocol.get_host_info().runtime_id;
let key_manager = state
.policy_verifier
.key_manager(ctx.clone(), &runtime_id, true)?;
.key_manager(ctx.clone(), &runtime_id)?;
let untrusted_policy = cbor::from_slice(&signed_policy_raw).map_err(|err| anyhow!(err))?;
let published_policy = state.policy_verifier.verify_key_manager_policy(
ctx,
untrusted_policy,
key_manager,
false,
)?;
let published_policy =
state
.policy_verifier
.verify_key_manager_policy(ctx, untrusted_policy, key_manager)?;

// Dispatch the local RPC call.
state
Expand Down Expand Up @@ -1058,14 +1056,11 @@ impl Dispatcher {
let runtime_id = state.protocol.get_host_info().runtime_id;
let key_manager = state
.policy_verifier
.key_manager(ctx.clone(), &runtime_id, true)?;
let policy = state.policy_verifier.verify_quote_policy(
ctx,
quote_policy,
&key_manager,
None,
false,
)?;
.key_manager(ctx.clone(), &runtime_id)?;
let policy =
state
.policy_verifier
.verify_quote_policy(ctx, quote_policy, &key_manager, None)?;

// Dispatch the local RPC call.
state.rpc_dispatcher.handle_km_quote_policy_update(policy);
Expand Down
49 changes: 13 additions & 36 deletions runtime/src/policy.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ use crate::{
registry::ImmutableState as RegistryState,
},
verifier::Verifier,
HEIGHT_LATEST,
},
};

Expand All @@ -30,8 +29,12 @@ pub enum PolicyVerifierError {
NoDeployment,
#[error("bad TEE constraints")]
BadTEEConstraints,
#[error("policy mismatch")]
PolicyMismatch,
#[error("policy hasn't been published")]
PolicyNotPublished,
#[error("status hasn't been published")]
StatusNotPublished,
#[error("configured runtime hardware mismatch")]
HardwareMismatch,
#[error("runtime doesn't use key manager")]
Expand Down Expand Up @@ -62,16 +65,9 @@ impl PolicyVerifier {
ctx: Arc<Context>,
runtime_id: &Namespace,
version: Option<Version>,
use_latest_state: bool,
) -> Result<QuotePolicy> {
// Verify to the latest height, if needed.
let consensus_state = if use_latest_state {
self.consensus_verifier.latest_state()?
} else {
self.consensus_verifier.state_at(HEIGHT_LATEST)?
};

// Fetch quote policy from the consensus layer using the given or the active version.
let consensus_state = self.consensus_verifier.latest_state()?;
let registry_state = RegistryState::new(&consensus_state);
let runtime = registry_state
.runtime(Context::create_child(&ctx), runtime_id)?
Expand Down Expand Up @@ -111,9 +107,8 @@ impl PolicyVerifier {
policy: QuotePolicy,
runtime_id: &Namespace,
version: Option<Version>,
use_latest_state: bool,
) -> Result<QuotePolicy> {
let published_policy = self.quote_policy(ctx, runtime_id, version, use_latest_state)?;
let published_policy = self.quote_policy(ctx, runtime_id, version)?;

if policy != published_policy {
debug!(
Expand All @@ -122,7 +117,7 @@ impl PolicyVerifier {
"untrusted" => ?policy,
"published" => ?published_policy,
);
return Err(PolicyVerifierError::PolicyNotPublished.into());
return Err(PolicyVerifierError::PolicyMismatch.into());
}

Ok(published_policy)
Expand All @@ -133,20 +128,13 @@ impl PolicyVerifier {
&self,
ctx: Arc<Context>,
key_manager: Namespace,
use_latest_state: bool,
) -> Result<SignedPolicySGX> {
// Verify to the latest height, if needed.
let consensus_state = if use_latest_state {
self.consensus_verifier.latest_state()?
} else {
self.consensus_verifier.state_at(HEIGHT_LATEST)?
};

// Fetch policy from the consensus layer.
let consensus_state = self.consensus_verifier.latest_state()?;
let km_state = KeyManagerState::new(&consensus_state);
let policy = km_state
.status(Context::create_child(&ctx), key_manager)?
.ok_or(PolicyVerifierError::PolicyNotPublished)?
.ok_or(PolicyVerifierError::StatusNotPublished)?
.policy
.ok_or(PolicyVerifierError::PolicyNotPublished)?;

Expand All @@ -159,9 +147,8 @@ impl PolicyVerifier {
ctx: Arc<Context>,
policy: SignedPolicySGX,
key_manager: Namespace,
use_latest_state: bool,
) -> Result<SignedPolicySGX> {
let published_policy = self.key_manager_policy(ctx, key_manager, use_latest_state)?;
let published_policy = self.key_manager_policy(ctx, key_manager)?;

if policy != published_policy {
debug!(
Expand All @@ -170,25 +157,15 @@ impl PolicyVerifier {
"untrusted" => ?policy,
"published" => ?published_policy,
);
return Err(PolicyVerifierError::PolicyNotPublished.into());
return Err(PolicyVerifierError::PolicyMismatch.into());
}

Ok(published_policy)
}

/// Fetch runtime's key manager.
pub fn key_manager(
&self,
ctx: Arc<Context>,
runtime_id: &Namespace,
use_latest_state: bool,
) -> Result<Namespace> {
let consensus_state = if use_latest_state {
self.consensus_verifier.latest_state()?
} else {
self.consensus_verifier.state_at(HEIGHT_LATEST)?
};

pub fn key_manager(&self, ctx: Arc<Context>, runtime_id: &Namespace) -> Result<Namespace> {
let consensus_state = self.consensus_verifier.latest_state()?;
let registry_state = RegistryState::new(&consensus_state);
let runtime = registry_state
.runtime(Context::create_child(&ctx), runtime_id)?
Expand Down

0 comments on commit d438583

Please sign in to comment.