diff --git a/.buildkite/go/nancy_audit.sh b/.buildkite/go/nancy_audit.sh index 7d1ca503779..e0e857bb119 100755 --- a/.buildkite/go/nancy_audit.sh +++ b/.buildkite/go/nancy_audit.sh @@ -15,6 +15,6 @@ set -euxo pipefail ######################################## # Check dependencies for vulnerabilities ######################################## -pushd go - go list -json -m all | nancy sleuth +pushd go/oasis-node + go list -json -deps | nancy sleuth popd diff --git a/.changelog/4709.feature.md b/.changelog/4709.feature.md new file mode 100644 index 00000000000..dfa7d47320e --- /dev/null +++ b/.changelog/4709.feature.md @@ -0,0 +1,7 @@ +runtime: Emit runtime logs as oasis-node logs + +Previously, runtime logs used a slightly different format. +Also, they were written to stdout in a manner that was not +synchronized with node logs, so the two sets of logs +sometimes intertwined mid-line. Those annoyances are gone, +plus runtime logs are now annotated with the runtime ID. diff --git a/.changelog/4754.internal.md b/.changelog/4754.internal.md new file mode 100644 index 00000000000..2a59fa81fb7 --- /dev/null +++ b/.changelog/4754.internal.md @@ -0,0 +1 @@ +docker/oasis-core-dev: Fix golangci-lint install diff --git a/.changelog/4757.feature.md b/.changelog/4757.feature.md new file mode 100644 index 00000000000..98da091da8a --- /dev/null +++ b/.changelog/4757.feature.md @@ -0,0 +1,4 @@ +runtime: Add support for reporting EnclaveRPC peer feedback + +This makes EnclaveRPC more robust as the higher-level layer in the +runtime can trigger peer replacement on high level errors. diff --git a/.changelog/4762.trivial.md b/.changelog/4762.trivial.md new file mode 100644 index 00000000000..e69de29bb2d diff --git a/client/src/enclave_rpc/client.rs b/client/src/enclave_rpc/client.rs index 1dc8b37ba53..e3a846f047c 100644 --- a/client/src/enclave_rpc/client.rs +++ b/client/src/enclave_rpc/client.rs @@ -53,7 +53,7 @@ pub enum RpcClientError { type SendqRequest = ( Arc, types::Request, - oneshot::Sender>, + oneshot::Sender>, usize, ); @@ -120,13 +120,7 @@ impl RpcClient { /// Construct an unconnected RPC client with runtime-internal transport. pub fn new_runtime(builder: Builder, protocol: Arc, endpoint: &str) -> Self { - Self::new( - Box::new(RuntimeTransport { - protocol, - endpoint: endpoint.to_owned(), - }), - builder, - ) + Self::new(Box::new(RuntimeTransport::new(protocol, endpoint)), builder) } /// Call a remote method. @@ -145,18 +139,27 @@ impl RpcClient { args: cbor::to_value(args), }; - let response = self.execute_call(ctx, request).await?; - match response.body { - types::Body::Success(value) => Ok(cbor::from_value(value)?), + let (pfid, response) = self.execute_call(ctx, request).await?; + let result = match response.body { + types::Body::Success(value) => cbor::from_value(value).map_err(Into::into), types::Body::Error(error) => Err(RpcClientError::CallFailed(error)), - } + }; + + // Report peer feedback based on whether call was successful. + let pf = match result { + Ok(_) => types::PeerFeedback::Success, + Err(_) => types::PeerFeedback::Failure, + }; + self.inner.transport.set_peer_feedback(pfid, Some(pf)); + + result } async fn execute_call( &self, ctx: Context, request: types::Request, - ) -> Result { + ) -> Result<(u64, types::Response), RpcClientError> { // Spawn a new controller if we haven't spawned one yet. if self .inner @@ -186,10 +189,23 @@ impl RpcClient { } .await; + // Update peer feedback for next request. + let pfid = inner.transport.get_peer_feedback_id(); + if result.is_err() { + // In case there was a transport error we need to reset the session + // immediately as no progress is possible. + let mut session = inner.session.lock().unwrap(); + session.reset(); + // Set peer feedback immediately so retries can try new peers. + inner + .transport + .set_peer_feedback(pfid, Some(types::PeerFeedback::Failure)); + } + match result { ref r if r.is_ok() || retries >= inner.max_retries => { // Request was successful or number of retries has been exceeded. - let _ = rsp_tx.send(result); + let _ = rsp_tx.send(result.map(|rsp| (pfid, rsp))); } _ => { @@ -373,6 +389,8 @@ mod test { rak: Arc, demux: Arc>, next_error: Arc, + peer_feedback: Arc)>>, + peer_feedback_history: Arc)>>>, } impl MockTransport { @@ -383,6 +401,8 @@ mod test { rak: rak.clone(), demux: Arc::new(Mutex::new(Demux::new(rak))), next_error: Arc::new(AtomicBool::new(false)), + peer_feedback: Arc::new(Mutex::new((0, None))), + peer_feedback_history: Arc::new(Mutex::new(Vec::new())), } } @@ -394,6 +414,17 @@ mod test { fn induce_transport_error(&self) { self.next_error.store(true, Ordering::SeqCst); } + + fn take_peer_feedback_history(&self) -> Vec<(u64, Option)> { + let mut pfh: Vec<_> = { + let mut pfh = self.peer_feedback_history.lock().unwrap(); + std::mem::take(&mut pfh) + }; + // Also add the pending feedback. + let pf = self.peer_feedback.lock().unwrap(); + pfh.push(pf.clone()); + pfh + } } impl Transport for MockTransport { @@ -402,9 +433,23 @@ mod test { _ctx: Context, data: Vec, ) -> BoxFuture, anyhow::Error>> { + let pf = { + let mut pf = self.peer_feedback.lock().unwrap(); + let peer_feedback = pf.1.take(); + + if !matches!(peer_feedback, None | Some(types::PeerFeedback::Success)) { + pf.0 += 1; + } + + (pf.0, peer_feedback) + }; + self.peer_feedback_history.lock().unwrap().push(pf); + + // Induce error when configured to do so. if self .next_error - .compare_and_swap(true, false, Ordering::SeqCst) + .compare_exchange(true, false, Ordering::SeqCst, Ordering::SeqCst) + .is_ok() { return Box::pin(future::err(anyhow!("transport error"))); } @@ -438,6 +483,19 @@ mod test { } } } + + fn set_peer_feedback(&self, pfid: u64, peer_feedback: Option) { + let mut pf = self.peer_feedback.lock().unwrap(); + if pf.0 != pfid { + return; + } + + pf.1 = peer_feedback; + } + + fn get_peer_feedback_id(&self) -> u64 { + self.peer_feedback.lock().unwrap().0 + } } #[test] @@ -454,6 +512,15 @@ mod test { .block_on(client.call(Context::background(), "test", 42)) .unwrap(); assert_eq!(result, 42, "call should work"); + assert_eq!( + transport.take_peer_feedback_history(), + vec![ + (0, None), // Handshake. + (0, None), // Handshake. + (0, None), // Call. + (0, Some(types::PeerFeedback::Success)), // Handled call. + ] + ); // Reset all sessions on the server and make sure that we can still get a response. transport.reset(); @@ -462,6 +529,16 @@ mod test { .block_on(client.call(Context::background(), "test", 43)) .unwrap(); assert_eq!(result, 43, "call should work"); + assert_eq!( + transport.take_peer_feedback_history(), + vec![ + (0, Some(types::PeerFeedback::Success)), // Previous handled call. + (1, Some(types::PeerFeedback::Failure)), // Failed call due to session reset. + (1, None), // New handshake. + (1, None), // New handshake. + (1, Some(types::PeerFeedback::Success)), // Handled call. + ] + ); // Induce a single transport error without resetting the server sessions and make sure we // can still get a response. @@ -471,5 +548,15 @@ mod test { .block_on(client.call(Context::background(), "test", 44)) .unwrap(); assert_eq!(result, 44, "call should work"); + assert_eq!( + transport.take_peer_feedback_history(), + vec![ + (1, Some(types::PeerFeedback::Success)), // Previous handled call. + (2, Some(types::PeerFeedback::Failure)), // Failed call due to induced error. + (2, None), // New handshake. + (2, None), // New handshake. + (2, Some(types::PeerFeedback::Success)), // Handled call. + ] + ); } } diff --git a/client/src/enclave_rpc/transport.rs b/client/src/enclave_rpc/transport.rs index aaec9bf2879..0d43b3ba07f 100644 --- a/client/src/enclave_rpc/transport.rs +++ b/client/src/enclave_rpc/transport.rs @@ -1,4 +1,4 @@ -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use anyhow::{anyhow, Error as AnyError}; use futures::future::{self, BoxFuture}; @@ -30,6 +30,15 @@ pub trait Transport: Send + Sync { ctx: Context, data: Vec, ) -> BoxFuture, AnyError>>; + + fn set_peer_feedback(&self, _pfid: u64, _peer_feedback: Option) { + // Default implementation doesn't do anything. + } + + fn get_peer_feedback_id(&self) -> u64 { + // Default implementation doesn't do anything. + 0 + } } /// A transport implementation which can be used from inside the runtime and uses the Runtime Host @@ -37,6 +46,18 @@ pub trait Transport: Send + Sync { pub struct RuntimeTransport { pub protocol: Arc, pub endpoint: String, + + peer_feedback: Mutex<(u64, Option)>, +} + +impl RuntimeTransport { + pub fn new(protocol: Arc, endpoint: &str) -> Self { + Self { + protocol, + endpoint: endpoint.to_string(), + peer_feedback: Mutex::new((0, None)), + } + } } impl Transport for RuntimeTransport { @@ -45,6 +66,20 @@ impl Transport for RuntimeTransport { ctx: Context, data: Vec, ) -> BoxFuture, AnyError>> { + let peer_feedback = { + let mut pf = self.peer_feedback.lock().unwrap(); + let peer_feedback = pf.1.take(); + + // If non-success feedback was propagated this means that the peer will be changed for + // subsequent requests. Increment pfid to make sure that we don't incorporate stale + // feedback. + if !matches!(peer_feedback, None | Some(types::PeerFeedback::Success)) { + pf.0 += 1; + } + + peer_feedback + }; + // NOTE: This is not actually async in SGX, but futures should be // dispatched on the current thread anyway. let rsp = self.protocol.call_host( @@ -52,6 +87,7 @@ impl Transport for RuntimeTransport { Body::HostRPCCallRequest { endpoint: self.endpoint.clone(), request: data, + peer_feedback, }, ); @@ -61,4 +97,17 @@ impl Transport for RuntimeTransport { Ok(_) => Box::pin(future::err(anyhow!("bad response type"))), } } + + fn set_peer_feedback(&self, pfid: u64, peer_feedback: Option) { + let mut pf = self.peer_feedback.lock().unwrap(); + if pf.0 != pfid { + return; + } + + pf.1 = peer_feedback; + } + + fn get_peer_feedback_id(&self) -> u64 { + self.peer_feedback.lock().unwrap().0 + } } diff --git a/docker/development/Dockerfile b/docker/development/Dockerfile index ecdd6d39663..ccbf33a425a 100644 --- a/docker/development/Dockerfile +++ b/docker/development/Dockerfile @@ -1,9 +1,9 @@ FROM ubuntu:20.04 # Package versions. -ARG GO_VERSION=1.17.7 -ARG GO_NANCY_VERSION=1.0.0 -ARG GO_NANCY_CHECKSUM=13804837a34c07e7a933b0d6f52c5e580b03ccb209e38fc3d6394b791b414c33 +ARG GO_VERSION=1.17.9 +ARG GO_NANCY_VERSION=1.0.33 +ARG GO_NANCY_CHECKSUM=a4bf5290d41b095c04f941ed5380674770c79d59735e33b1bd07a5cd5fbb135d ARG GO_PROTOC_VERSION=3.6.1 ARG GO_PROTOC_GEN_GO_VERSION=1.21.0 ARG GOLANGCILINT_VERSION=1.41.1 @@ -61,12 +61,12 @@ RUN wget https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz && \ # Install a specific version of protoc-gen-go. go install google.golang.org/protobuf/cmd/protoc-gen-go@v${GO_PROTOC_GEN_GO_VERSION} && \ # Install golangci-lint. - curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | bash -s -- -b /tmp/bin v${GOLANGCILINT_VERSION} && \ + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b /tmp/bin v${GOLANGCILINT_VERSION} && \ mv /tmp/bin/golangci-lint /go/bin && \ # Install gocovmerge for e2e coverage. go install github.com/wadey/gocovmerge@${GOCOVMERGE_VERSION} && \ # Install nancy for auditing dependencies. - curl -sfL -o nancy https://github.com/sonatype-nexus-community/nancy/releases/download/v${GO_NANCY_VERSION}/nancy-linux.amd64-v${GO_NANCY_VERSION} && \ + curl -sfL -o nancy https://github.com/sonatype-nexus-community/nancy/releases/download/v${GO_NANCY_VERSION}/nancy-v${GO_NANCY_VERSION}-linux-amd64 && \ echo "${GO_NANCY_CHECKSUM} nancy" | sha256sum -c && \ mv nancy /go/bin/nancy && \ chmod +x /go/bin/nancy && \ diff --git a/go/.nancy-ignore b/go/.nancy-ignore index 8a69be6389e..e69de29bb2d 100644 --- a/go/.nancy-ignore +++ b/go/.nancy-ignore @@ -1 +0,0 @@ -CVE-2020-26160 # Until viper and etcd/prometheus are upgraded to not need jwt-go. diff --git a/go/common/logging/logging.go b/go/common/logging/logging.go index 315b57f72f5..2a47620c91b 100644 --- a/go/common/logging/logging.go +++ b/go/common/logging/logging.go @@ -213,6 +213,19 @@ func GetLoggerEx(module string, extraUnwind int) *Logger { return backend.getLogger(module, extraUnwind) } +// GetBaseLogger creates a new non-prefixed logger instance with the +// specified module. +// +// The returned logger will not pre-include any log fields (aka prefixes) +// except for the module name. Its level will be set in accordance +// with the global config, which can be per-module. +// +// This may be called from any point, including before Initialize is +// called, allowing for the construction of a package level Logger. +func GetBaseLogger(module string) *Logger { + return backend.getBaseLogger(module) +} + // Initialize initializes the logging backend to write to the provided // Writer with the given format and log levels specified for each // module. If the requested module is not given, default level is @@ -239,7 +252,6 @@ func Initialize(w io.Writer, format Format, defaultLvl Level, moduleLvls map[str } logger = level.NewFilter(logger, defaultLvl.toOption()) - logger = log.With(logger, "ts", log.DefaultTimestampUTC) backend.baseLogger = logger backend.moduleLevels = moduleLvls @@ -260,7 +272,7 @@ func Initialize(w io.Writer, format Format, defaultLvl Level, moduleLvls map[str // libp2p/IPFS uses yet another logging library, that appears to be a // wrapper around zap. - ipfsLogger := newZapCore(logger, "libp2p", 7) + ipfsLogger := newZapCore(log.With(logger, "ts", log.DefaultTimestampUTC), "libp2p", 7) backend.setupLogLevelLocked(ipfsLogger.logger) // Update the ipfs core logger. @@ -323,19 +335,38 @@ func (b *logBackend) getLogger(module string, extraUnwind int) *Logger { logger = &log.SwapLogger{} } - var keyvals []interface{} - if module != "" { - keyvals = append(keyvals, []interface{}{ - "module", - module, - }...) + prefixes := []interface{}{ + "ts", log.DefaultTimestampUTC, + "caller", log.Caller(defaultUnwind + extraUnwind), + "module", module, } - keyvals = append(keyvals, []interface{}{ - "caller", - log.Caller(defaultUnwind + extraUnwind), - }...) l := &Logger{ - logger: log.WithPrefix(logger, keyvals...), + logger: log.WithPrefix(logger, prefixes...), + module: module, + } + b.setupLogLevelLocked(l) + + if !b.initialized { + // Stash the logger so that it can be instantiated once logging + // is actually initialized. + sLog := logger.(*log.SwapLogger) + b.earlyLoggers = append(b.earlyLoggers, &earlyLogger{swapLogger: sLog, logger: l}) + } + + return l +} + +func (b *logBackend) getBaseLogger(module string) *Logger { + b.Lock() + defer b.Unlock() + + logger := b.baseLogger + if !b.initialized { + logger = &log.SwapLogger{} + } + + l := &Logger{ + logger: log.WithPrefix(logger, "module", module), module: module, } b.setupLogLevelLocked(l) diff --git a/go/go.mod b/go/go.mod index e6c6bfd0a49..1aa3232acfc 100644 --- a/go/go.mod +++ b/go/go.mod @@ -1,11 +1,6 @@ module github.com/oasisprotocol/oasis-core/go replace ( - // Fixes vulnerabilities in etcd v3.3.{10,13} (dependencies via viper). - // Can be removed once there is a spf13/viper release with updated - // etcd and other dependencies using viper are updated. - // https://github.com/spf13/viper/issues/956 - github.com/coreos/etcd => github.com/coreos/etcd v3.3.25+incompatible // Updates the version used by badgerdb, because some of the Go // module caches apparently have a messed up copy that causes // build failures. @@ -27,13 +22,13 @@ require ( github.com/dgraph-io/badger/v3 v3.2103.2 github.com/eapache/channels v1.1.0 github.com/fxamacker/cbor/v2 v2.4.0 - github.com/go-kit/log v0.2.0 + github.com/go-kit/log v0.2.1 github.com/golang/protobuf v1.5.2 github.com/golang/snappy v0.0.4 github.com/google/btree v1.0.1 github.com/hashicorp/go-hclog v1.1.0 github.com/hashicorp/go-multierror v1.1.1 - github.com/hashicorp/go-plugin v1.4.3 + github.com/hashicorp/go-plugin v1.4.4 github.com/hpcloud/tail v1.0.0 github.com/ianbruene/go-difflib v1.2.0 github.com/ipfs/go-log/v2 v2.5.0 @@ -45,8 +40,8 @@ require ( github.com/oasisprotocol/deoxysii v0.0.0-20220228165953-2091330c22b7 github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5 github.com/powerman/rpc-codec v1.2.2 - github.com/prometheus/client_golang v1.12.1 - github.com/prometheus/common v0.32.1 + github.com/prometheus/client_golang v1.12.2 + github.com/prometheus/common v0.34.0 github.com/prometheus/procfs v0.7.3 github.com/seccomp/libseccomp-golang v0.9.1 github.com/spf13/cobra v1.3.0 @@ -57,14 +52,14 @@ require ( github.com/tendermint/tm-db v0.6.6 github.com/thepudds/fzgo v0.2.2 github.com/tyler-smith/go-bip39 v1.1.0 - go.uber.org/multierr v1.7.0 - go.uber.org/zap v1.19.1 - golang.org/x/crypto v0.0.0-20210915214749-c084706c2272 - golang.org/x/net v0.0.0-20211005001312-d4b1ae081e3b - google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa - google.golang.org/grpc v1.45.0 + go.uber.org/multierr v1.8.0 + go.uber.org/zap v1.21.0 + golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 + golang.org/x/net v0.0.0-20220418201149-a630d4f3e7a2 + google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac + google.golang.org/grpc v1.46.2 google.golang.org/grpc/security/advancedtls v0.0.0-20200902210233-8630cac324bf - google.golang.org/protobuf v1.27.1 + google.golang.org/protobuf v1.28.0 ) require ( diff --git a/go/go.sum b/go/go.sum index cae6f1256f2..7c86587c098 100644 --- a/go/go.sum +++ b/go/go.sum @@ -186,7 +186,7 @@ github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8a github.com/containerd/continuity v0.2.0 h1:j/9Wnn+hrEWjLvHuIxUU1YI5JjEjVlT2AA68cse9rwY= github.com/containerd/continuity v0.2.0/go.mod h1:wCYX+dRqZdImhGucXOqTQn05AhX6EUDaGEMUzTFFpLg= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.25+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= @@ -257,6 +257,7 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.m github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/go-control-plane v0.10.1/go.mod h1:AY7fTTXNdv/aJ2O5jwpxAPOWUZ7hQAEvzN5Pf27BkQQ= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E9/baC+qXE/TeeyBRzgJDws= github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64= @@ -304,8 +305,9 @@ github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgO github.com/go-kit/kit v0.12.0 h1:e4o3o3IsBfAKQh5Qbbiqyfu97Ku7jrO/JbohvztANh4= github.com/go-kit/kit v0.12.0/go.mod h1:lHd+EkCZPIwYItmGDDRdhinkzX2A1sj+M9biaEaizzs= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-kit/log v0.2.0 h1:7i2K3eKTos3Vc0enKCfnVcgHh2olr/MyfboYq7cAcFw= github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= +github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= +github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= @@ -481,8 +483,8 @@ github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHh github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/hashicorp/go-plugin v1.4.3 h1:DXmvivbWD5qdiBts9TpBC7BYL1Aia5sxbRgQB+v6UZM= -github.com/hashicorp/go-plugin v1.4.3/go.mod h1:5fGEH17QVwTTcR0zV7yhDPLLmFX9YSZ38b18Udy6vYQ= +github.com/hashicorp/go-plugin v1.4.4 h1:NVdrSdFRt3SkZtNckJ6tog7gbpRrcbOjQi/rgF7JYWQ= +github.com/hashicorp/go-plugin v1.4.4/go.mod h1:viDMjcLJuDui6pXb8U4HVfb8AamCWhHGUjr2IrTF67s= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= @@ -1009,8 +1011,9 @@ github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP github.com/prometheus/client_golang v1.9.0/go.mod h1:FqZLKOZnGdFAhOK4nqGHa7D66IdsO+O441Eve7ptJDU= github.com/prometheus/client_golang v1.10.0/go.mod h1:WJM3cc3yu7XKBKa/I8WeZm+V3eltZnBwfENSU7mdogU= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_golang v1.12.2 h1:51L9cDoUHVrXx4zWYlcLQIZ+d+VXHgqnYKkIuq4g/34= +github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -1030,8 +1033,9 @@ github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16 github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.30.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/common v0.34.0 h1:RBmGO9d/FVjqHT0yUGQwBJhkwKV+wPCn7KGpvfab0uE= +github.com/prometheus/common v0.34.0/go.mod h1:gB3sOl7P0TvJabZpLY5uQMpUqRCPPCyRLCZYc7JZTNE= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= @@ -1235,14 +1239,16 @@ go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/goleak v1.1.11-0.20210813005559-691160354723 h1:sHOAIxRGBp443oHZIPB+HsUGaksVCXVQENPxwTfQdH4= go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= +go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec= go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= +go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= +go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= @@ -1250,8 +1256,9 @@ go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= -go.uber.org/zap v1.19.1 h1:ue41HOKd1vGURxrmeKIgELGb3jPW9DMUDGtsinblHwI= go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= +go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8= +go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -1285,8 +1292,9 @@ golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf/go.mod h1:P+XmwS30IXTQdn5 golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210813211128-0a44fdfbc16e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20210915214749-c084706c2272 h1:3erb+vDS8lU1sxfDHF4/hhWyaXnhIaO+7RgL4fDZORA= golang.org/x/crypto v0.0.0-20210915214749-c084706c2272/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 h1:kUhD7nTDoI3fVd9G4ORWrbV5NY0liEs/Jg2pv5f+bBA= +golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1392,8 +1400,12 @@ golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210917221730-978cfadd31cf/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211005001312-d4b1ae081e3b h1:SXy8Ld8oKlcogOvUAh0J5Pm5RKzgYBMMxLxt6n5XW50= golang.org/x/net v0.0.0-20211005001312-d4b1ae081e3b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220418201149-a630d4f3e7a2 h1:6mzvA99KwZxbOrxww4EvWVQUnN1+xEu9tafK5ZxkYeA= +golang.org/x/net v0.0.0-20220418201149-a630d4f3e7a2/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1413,8 +1425,9 @@ golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 h1:RerP+noqYHUQ8CMRcPlC2nvTa4dcBIjegkuWdcUDuqg= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b h1:clP8eMhB30EHdc0bd2Twtq6kgU7yl5ub2cQLSdrv1Dg= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1536,11 +1549,13 @@ golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 h1:nhht2DYV/Sn3qOayu8lM+cU1ii9sTLUeBQwQQfUHtrs= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1754,8 +1769,9 @@ google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ6 google.golang.org/genproto v0.0.0-20211129164237-f09f9a12af12/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211203200212-54befc351ae9/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa h1:I0YcKz0I7OAhddo7ya8kMnvprhcWM045PmkBdMO9zN0= google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac h1:qSNTkEN+L2mvWcLgJOR+8bdHX9rN/IdU3A1Ghpfb1Rg= +google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= @@ -1793,8 +1809,9 @@ google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9K google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.45.0 h1:NEpgUqV3Z+ZjkqMsxMg11IaDrXY4RY6CQukSGK0uI1M= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= +google.golang.org/grpc v1.46.2 h1:u+MLGgVf7vRdjEYZ8wDFhAVNmhkbJ5hmrA1LMWK1CAQ= +google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/grpc/examples v0.0.0-20200731180010-8bec2f5d898f h1:HNNmM2dnxUknBEJDvuCRZcUzhGbhkz00ckV2ha2gFJY= google.golang.org/grpc/examples v0.0.0-20200731180010-8bec2f5d898f/go.mod h1:TGiSRL2BBv2WqzfsFNWYp/pkWdtf5kbZS/DQ9Ee3mWk= @@ -1812,8 +1829,9 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/go/registry/api/runtime.go b/go/registry/api/runtime.go index b77adf34a8f..f484b6c7f8d 100644 --- a/go/registry/api/runtime.go +++ b/go/registry/api/runtime.go @@ -532,6 +532,11 @@ func (r *Runtime) ValidateDeployments(now beacon.EpochTime) error { if len(r.Deployments) > 2 { return fmt.Errorf("%w: too many deployments", ErrInvalidArgument) } + for _, d := range r.Deployments { + if d == nil { + return fmt.Errorf("%w: nil deployment", ErrInvalidArgument) + } + } deployments := make([]*VersionInfo, len(r.Deployments)) copy(deployments, r.Deployments) diff --git a/go/runtime/enclaverpc/api/api.go b/go/runtime/enclaverpc/api/api.go index b7e16aa4237..a6a8e757697 100644 --- a/go/runtime/enclaverpc/api/api.go +++ b/go/runtime/enclaverpc/api/api.go @@ -3,9 +3,32 @@ package api // Frame is an EnclaveRPC frame. // -// It is the Go analog of the Rust RPC frame defined in client/src/rpc/types.rs. +// It is the Go analog of the Rust RPC frame defined in runtime/src/enclave_rpc/types.rs. type Frame struct { Session []byte `json:"session,omitempty"` UntrustedPlaintext string `json:"untrusted_plaintext,omitempty"` Payload []byte `json:"payload,omitempty"` } + +// PeerFeedback is the feedback on the peer that handled the last RPC call. +type PeerFeedback uint8 + +const ( + PeerFeedbackSuccess PeerFeedback = 0 + PeerFeedbackFailure PeerFeedback = 1 + PeerFeedbackBadPeer PeerFeedback = 2 +) + +// String returns a string representation of peer feedback. +func (pf PeerFeedback) String() string { + switch pf { + case PeerFeedbackSuccess: + return "success" + case PeerFeedbackFailure: + return "failure" + case PeerFeedbackBadPeer: + return "bad peer" + default: + return "[unknown]" + } +} diff --git a/go/runtime/host/logger.go b/go/runtime/host/logger.go new file mode 100644 index 00000000000..8a27686c6e5 --- /dev/null +++ b/go/runtime/host/logger.go @@ -0,0 +1,128 @@ +package host + +import ( + "encoding/json" + "strings" + + "github.com/oasisprotocol/oasis-core/go/common/logging" +) + +// Max number of bytes to buffer in the runtime log wrapper, i.e. roughly +// the longest expected valid log line from the runtime. +const maxLogBufferSize = 10_000_000 + +// RuntimeLogWrapper is a Writer that interprets data written to it as JSON-formatted +// runtime logs, and re-logs the messages as oasis-node logs. For example, it +// translates runtime log levels to oasis-node log levels, because the two have +// slightly different formats. +// +// It hardcodes some assumptions about the format of the runtime logs. +type RuntimeLogWrapper struct { + // Logger for wrapper-internal info/errors. + logger *logging.Logger + // Loggers for the runtime, one for each module inside the runtime. + rtLoggers map[string]*logging.Logger + // Key-value pairs to append to each log entry. + suffixes []interface{} + // Buffer for accumulating incoming log entries from the runtime. + buf []byte +} + +// NewRuntimeLogWrapper creates a new RuntimeLogWrapper. +func NewRuntimeLogWrapper(logger *logging.Logger, suffixes ...interface{}) *RuntimeLogWrapper { + return &RuntimeLogWrapper{ + logger: logger, + suffixes: suffixes, + rtLoggers: make(map[string]*logging.Logger), + } +} + +// Implements io.Writer +func (w *RuntimeLogWrapper) Write(chunk []byte) (int, error) { + w.buf = append(w.buf, chunk...) + + // Find and process any full lines that have accumulated in the buffer. + // We assume one line per log entry. + for i := len(w.buf) - len(chunk); i < len(w.buf); i++ { + if w.buf[i] == '\n' { + w.processLogLine(w.buf[:i]) + w.buf = w.buf[i+1:] + i = 0 + } + } + + // Prevent the buffer from growing indefinitely in case runtime logs + // don't contain newlines (e.g. because of unexpected log format). + if len(w.buf) > maxLogBufferSize { + w.logger.Warn("runtime log buffer is too large, dropping logs") + w.buf = w.buf[:0] + } + + // Always report success. Even if log lines were malformed, we processed them + // and reported the malformedness. + return len(chunk), nil +} + +// rtLogger returns the logger for the given module, creating it if needed. +func (w *RuntimeLogWrapper) rtLogger(module string) *logging.Logger { + if l, ok := w.rtLoggers[module]; ok { + return l + } + l := logging.GetBaseLogger(module).With(w.suffixes...) + w.rtLoggers[module] = l + return l +} + +func (w RuntimeLogWrapper) processLogLine(line []byte) { + // Interpret line as JSON. + var m map[string]interface{} + if err := json.Unmarshal(line, &m); err != nil { + w.logger.Warn("non-JSON log line from runtime", "log_line", string(line), "err", err) + return + } + + // Destructure JSON into key-value pairs, parse common fields. + var kv []interface{} + var msg string + var level string + var module string + for k, v := range m { + if k == "msg" { + if _msg, ok := v.(string); ok { + msg = _msg + } else { + w.logger.Warn("malformed log line from runtime", "log_line", string(line), "err", "msg is not a string") + return + } + } else if k == "level" { + level, _ = v.(string) + } else if k == "module" { + module, _ = v.(string) + if module == "" { + module = "runtime" + } + // Enforce "runtime" scope in the module name. + if !(module == "runtime" || strings.HasPrefix(module, "runtime/")) { + module = "runtime/" + module + } + } else { + kv = append(kv, k, v) + } + } + + // Output the log. + rtLogger := w.rtLogger(module) + switch level { + case "DEBG": + rtLogger.Debug(msg, kv...) + case "INFO": + rtLogger.Info(msg, kv...) + case "WARN": + rtLogger.Warn(msg, kv...) + case "ERRO": + rtLogger.Error(msg, kv...) + default: + w.logger.Warn("log line from runtime has no known error level set, using INFO", "log_line", string(line)) + rtLogger.Info(msg, kv...) + } +} diff --git a/go/runtime/host/logger_test.go b/go/runtime/host/logger_test.go new file mode 100644 index 00000000000..cb807e7bf8b --- /dev/null +++ b/go/runtime/host/logger_test.go @@ -0,0 +1,67 @@ +package host + +import ( + "bytes" + "strings" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/oasisprotocol/oasis-core/go/common/logging" +) + +func TestRuntimeLogWrapper(t *testing.T) { + require := require.New(t) + + // Redirect all logs to a buffer where we can inspect them later. + // Use JSON format because it uses a deterministic (a-z) order of keys. + var buf bytes.Buffer + _ = logging.Initialize(&buf, logging.FmtJSON, logging.LevelDebug, map[string]logging.Level{}) + + // Simulated runtime output. + logChunks := []string{ + // A message in multiple chunks + `{"msg":"Runtime is starti`, `ng","level":"INFO","ts":"2022-04-26","module":"runtime"} + `, + // A message with multiple chunks, one of them being just the terminating newline + `{`, `"msg":"My info\nwith a newline","level":"INFO","ts":"2022","module":"runtime"}`, "\n", + // A chunk containing multiple messages + `{"msg":"My debug","level":"DEBG","ts":"2022-04-27","module":"runtime/dispatcher"} + {"msg":"My error","lev`, `e`, `l":"ERRO","ts":"2022-04-28","module":"runtime/protocol","err":"some explanation"} + `, + // Malformed (non-JSON) output + "Random crap\n", + // One more valid JSON to make sure we've recovered. Also tests non-"runtime"-scoped module name. + `{"msg":"Should be recovered","level":"INFO","ts":"2022","module":"foo"`, "}\n", + } + + // Feed data to RuntimeLogWrapper. + w := NewRuntimeLogWrapper(logging.GetLogger("testenv")) + for _, chunk := range logChunks { + n, err := w.Write([]byte(chunk)) + require.Equal(len(chunk), n) + require.NoError(err) + } + + actual := strings.Split(buf.String(), "\n") + expected := []string{ + `{"level":"info","module":"runtime","msg":"Runtime is starting","ts":"2022-04-26"}`, + `{"level":"info","module":"runtime","msg":"My info\\nwith a newline","ts":"2022"}`, + `{"level":"debug","module":"runtime/dispatcher","msg":"My debug","ts":"2022-04-27"}`, + `{"err":"some explanation","level":"error","module":"runtime/protocol","msg":"My error","ts":"2022-04-28"}`, + `{"caller":"logger.go:\d+","err":"invalid character 'R' looking for beginning of value","level":"warn","log_line":"\\t\\tRandom crap","module":"testenv","msg":"non-JSON log line from runtime","ts":"[^"]+"}`, + `{"level":"info","module":"runtime/foo","msg":"Should be recovered","ts":"2022"}`, + ``, // Because we split on newline and the last log entry ends with a newline + } + + require.EqualValues( + len(expected), len(actual), + "Unexpected number of log entries; expected %d, got %d: %#v", + len(expected), len(actual), actual) + for i := range actual { + require.Regexp( + expected[i], actual[i], + "Log line %2d was %#v\nbut should match regex %#v", + i+1, actual[i], expected[i]) + } +} diff --git a/go/runtime/host/protocol/types.go b/go/runtime/host/protocol/types.go index 5f8ed058249..8b4ea40dcba 100644 --- a/go/runtime/host/protocol/types.go +++ b/go/runtime/host/protocol/types.go @@ -16,6 +16,7 @@ import ( "github.com/oasisprotocol/oasis-core/go/roothash/api/block" "github.com/oasisprotocol/oasis-core/go/roothash/api/commitment" "github.com/oasisprotocol/oasis-core/go/roothash/api/message" + enclaverpc "github.com/oasisprotocol/oasis-core/go/runtime/enclaverpc/api" "github.com/oasisprotocol/oasis-core/go/runtime/transaction" storage "github.com/oasisprotocol/oasis-core/go/storage/api" ) @@ -405,6 +406,14 @@ type RuntimeConsensusSyncRequest struct { type HostRPCCallRequest struct { Endpoint string `json:"endpoint"` Request []byte `json:"request"` + + // PeerFeedback contains optional peer feedback for the last RPC call under the given endpoint. + // + // This enables the runtime to notify the node whether the given peer should continue to be used + // or not based on higher-level logic that lives in the runtime. + // + // In case no feedback is given success is assumed. + PeerFeedback *enclaverpc.PeerFeedback `json:"pf,omitempty"` } // HostRPCCallResponse is a host RPC call response message body. diff --git a/go/runtime/host/sandbox/sandbox.go b/go/runtime/host/sandbox/sandbox.go index 54a32c4f25d..7568bf04e3a 100644 --- a/go/runtime/host/sandbox/sandbox.go +++ b/go/runtime/host/sandbox/sandbox.go @@ -525,15 +525,26 @@ func (r *sandboxedRuntime) manager() { // New creates a new runtime provisioner that uses a local process sandbox. func New(cfg Config) (host.Provisioner, error) { + // Use a default Logger if none was provided. + if cfg.Logger == nil { + cfg.Logger = logging.GetLogger("runtime/host/sandbox") + } // Use a default GetSandboxConfig if none was provided. if cfg.GetSandboxConfig == nil { cfg.GetSandboxConfig = func(hostCfg host.Config, socketPath, runtimeDir string) (process.Config, error) { + logWrapper := host.NewRuntimeLogWrapper( + cfg.Logger, + "runtime_id", hostCfg.Bundle.Manifest.ID, + "runtime_name", hostCfg.Bundle.Manifest.Name, + ) return process.Config{ Path: hostCfg.Bundle.Path, Env: map[string]string{ "OASIS_WORKER_HOST": socketPath, }, SandboxBinaryPath: cfg.SandboxBinaryPath, + Stdout: logWrapper, + Stderr: logWrapper, }, nil } } @@ -555,9 +566,5 @@ func New(cfg Config) (host.Provisioner, error) { }, nil } } - // Use a default Logger if none was provided. - if cfg.Logger == nil { - cfg.Logger = logging.GetLogger("runtime/host/sandbox") - } return &provisioner{cfg: cfg}, nil } diff --git a/go/runtime/host/sgx/sgx.go b/go/runtime/host/sgx/sgx.go index cff1672c7b2..9d81a64e0ae 100644 --- a/go/runtime/host/sgx/sgx.go +++ b/go/runtime/host/sgx/sgx.go @@ -185,6 +185,12 @@ func (s *sgxProvisioner) getSandboxConfig(rtCfg host.Config, socketPath, runtime } s.logger.Info("found SGX device", "path", sgxDev) + logWrapper := host.NewRuntimeLogWrapper( + s.logger, + "runtime_id", rtCfg.Bundle.Manifest.ID, + "runtime_name", rtCfg.Bundle.Manifest.Name, + ) + return process.Config{ Path: s.cfg.LoaderPath, Args: []string{ @@ -204,6 +210,8 @@ func (s *sgxProvisioner) getSandboxConfig(rtCfg host.Config, socketPath, runtime signaturePath: bytes.NewReader(sig), }, SandboxBinaryPath: s.cfg.SandboxBinaryPath, + Stdout: logWrapper, + Stderr: logWrapper, }, nil } diff --git a/go/runtime/keymanager/api/api.go b/go/runtime/keymanager/api/api.go index 80bf8a02f48..adedc72c0c2 100644 --- a/go/runtime/keymanager/api/api.go +++ b/go/runtime/keymanager/api/api.go @@ -1,7 +1,11 @@ // Package api defines the key manager client API. package api -import "context" +import ( + "context" + + enclaverpc "github.com/oasisprotocol/oasis-core/go/runtime/enclaverpc/api" +) // EnclaveRPCEndpoint is the name of the key manager EnclaveRPC endpoint. const EnclaveRPCEndpoint = "key-manager" @@ -9,5 +13,8 @@ const EnclaveRPCEndpoint = "key-manager" // Client is the key manager client interface. type Client interface { // CallEnclave calls the key manager via remote EnclaveRPC. - CallEnclave(ctx context.Context, data []byte) ([]byte, error) + // + // The provided peer feedback is optional feedback on the peer that handled the last EnclaveRPC + // request (if any) which may be used to inform the routing decision. + CallEnclave(ctx context.Context, data []byte, pf *enclaverpc.PeerFeedback) ([]byte, error) } diff --git a/go/runtime/registry/host.go b/go/runtime/registry/host.go index cb31d810ddc..01cbd430303 100644 --- a/go/runtime/registry/host.go +++ b/go/runtime/registry/host.go @@ -173,7 +173,7 @@ func (h *runtimeHostHandler) Handle(ctx context.Context, body *protocol.Body) (* if err != nil { return nil, err } - res, err := kmCli.CallEnclave(ctx, body.HostRPCCallRequest.Request) + res, err := kmCli.CallEnclave(ctx, body.HostRPCCallRequest.Request, body.HostRPCCallRequest.PeerFeedback) if err != nil { return nil, err } diff --git a/go/worker/common/committee/keymanager.go b/go/worker/common/committee/keymanager.go index cffe3e57db0..866d14c8904 100644 --- a/go/worker/common/committee/keymanager.go +++ b/go/worker/common/committee/keymanager.go @@ -6,23 +6,34 @@ import ( "sync" "github.com/oasisprotocol/oasis-core/go/common" + "github.com/oasisprotocol/oasis-core/go/common/logging" + consensus "github.com/oasisprotocol/oasis-core/go/consensus/api" + enclaverpc "github.com/oasisprotocol/oasis-core/go/runtime/enclaverpc/api" + "github.com/oasisprotocol/oasis-core/go/worker/common/p2p" + "github.com/oasisprotocol/oasis-core/go/worker/common/p2p/rpc" keymanagerP2P "github.com/oasisprotocol/oasis-core/go/worker/keymanager/p2p" ) // KeyManagerClientWrapper is a wrapper for the key manager P2P client that handles deferred // initialization after the key manager runtime ID is known. +// +// It also handles peer feedback propagation from EnclaveRPC in the runtime. type KeyManagerClientWrapper struct { - l sync.RWMutex + l sync.Mutex - id *common.Namespace - n *Node - cli keymanagerP2P.Client + id *common.Namespace + p2p *p2p.P2P + consensus consensus.Backend + cli keymanagerP2P.Client + logger *logging.Logger + + lastPeerFeedback rpc.PeerFeedback } // Initialized returns a channel that gets closed when the client is initialized. func (km *KeyManagerClientWrapper) Initialized() <-chan struct{} { - km.l.RLock() - defer km.l.RUnlock() + km.l.Lock() + defer km.l.Unlock() // If no active key manager client, return a closed channel. if km.cli == nil { @@ -34,7 +45,8 @@ func (km *KeyManagerClientWrapper) Initialized() <-chan struct{} { return km.cli.Initialized() } -func (km *KeyManagerClientWrapper) setKeymanagerID(id *common.Namespace) { +// SetKeyManagerID configures the key manager runtime ID to use. +func (km *KeyManagerClientWrapper) SetKeyManagerID(id *common.Namespace) { km.l.Lock() defer km.l.Unlock() @@ -43,7 +55,7 @@ func (km *KeyManagerClientWrapper) setKeymanagerID(id *common.Namespace) { return } - km.n.logger.Debug("key manager updated", + km.logger.Debug("key manager updated", "keymanager_id", id, ) km.id = id @@ -54,33 +66,72 @@ func (km *KeyManagerClientWrapper) setKeymanagerID(id *common.Namespace) { } if id != nil { - km.cli = keymanagerP2P.NewClient(km.n.P2P, km.n.Consensus, *id) + km.cli = keymanagerP2P.NewClient(km.p2p, km.consensus, *id) } + + km.lastPeerFeedback = nil } // Implements runtimeKeymanager.Client. -func (km *KeyManagerClientWrapper) CallEnclave(ctx context.Context, data []byte) ([]byte, error) { - km.l.RLock() +func (km *KeyManagerClientWrapper) CallEnclave( + ctx context.Context, + data []byte, + pf *enclaverpc.PeerFeedback, +) ([]byte, error) { + km.l.Lock() cli := km.cli - km.l.RUnlock() + lastPf := km.lastPeerFeedback + km.l.Unlock() if cli == nil { return nil, fmt.Errorf("key manager not available") } - rsp, pf, err := km.cli.CallEnclave(ctx, &keymanagerP2P.CallEnclaveRequest{ + // Propagate peer feedback on the last EnclaveRPC call to guide routing decision. + if lastPf != nil { + // If no feedback has been provided by the runtime, treat previous call as success. + if pf == nil { + pfv := enclaverpc.PeerFeedbackSuccess + pf = &pfv + } + + km.logger.Debug("received peer feedback from runtime", + "peer_feedback", *pf, + ) + + switch *pf { + case enclaverpc.PeerFeedbackSuccess: + lastPf.RecordSuccess() + case enclaverpc.PeerFeedbackFailure: + lastPf.RecordFailure() + case enclaverpc.PeerFeedbackBadPeer: + lastPf.RecordBadPeer() + default: + } + } + + rsp, nextPf, err := cli.CallEnclave(ctx, &keymanagerP2P.CallEnclaveRequest{ Data: data, }) if err != nil { return nil, err } - // TODO: Support reporting peer feedback from the enclave. - pf.RecordSuccess() + + // Store peer feedback instance that we can use. + km.l.Lock() + if km.cli == cli { // Key manager could get updated while we are doing the call. + km.lastPeerFeedback = nextPf + } + km.l.Unlock() + return rsp.Data, nil } -func newKeyManagerClientWrapper(n *Node) *KeyManagerClientWrapper { +// NewKeyManagerClientWrapper creates a new key manager client wrapper. +func NewKeyManagerClientWrapper(p2p *p2p.P2P, consensus consensus.Backend, logger *logging.Logger) *KeyManagerClientWrapper { return &KeyManagerClientWrapper{ - n: n, + p2p: p2p, + consensus: consensus, + logger: logger, } } diff --git a/go/worker/common/committee/node.go b/go/worker/common/committee/node.go index c89300e24b8..c24daa75cf2 100644 --- a/go/worker/common/committee/node.go +++ b/go/worker/common/committee/node.go @@ -215,7 +215,7 @@ func (n *Node) Stop() { n.stopOnce.Do(func() { close(n.stopCh) n.TxPool.Stop() - n.KeyManagerClient.setKeymanagerID(nil) + n.KeyManagerClient.SetKeyManagerID(nil) }) } @@ -451,7 +451,7 @@ func (n *Node) handleNewBlockLocked(blk *block.Block, height int64) { n.updateHostedRuntimeVersionLocked() // Make sure to update the key manager if needed. - n.KeyManagerClient.setKeymanagerID(n.CurrentDescriptor.KeyManager) + n.KeyManagerClient.SetKeyManagerID(n.CurrentDescriptor.KeyManager) } for _, hooks := range n.hooks { @@ -569,7 +569,7 @@ func (n *Node) worker() { "keymanager_runtime_id", *rt.KeyManager, ) - n.KeyManagerClient.setKeymanagerID(rt.KeyManager) + n.KeyManagerClient.SetKeyManagerID(rt.KeyManager) select { case <-n.ctx.Done(): n.logger.Error("failed to wait for key manager", @@ -819,7 +819,7 @@ func NewNode( } // Prepare the key manager client wrapper. - n.KeyManagerClient = newKeyManagerClientWrapper(n) + n.KeyManagerClient = NewKeyManagerClientWrapper(p2pHost, consensus, n.logger) // Prepare the runtime host node helpers. rhn, err := runtimeRegistry.NewRuntimeHostNode(n) diff --git a/go/worker/keymanager/handler.go b/go/worker/keymanager/handler.go index 999deb96292..747a160966c 100644 --- a/go/worker/keymanager/handler.go +++ b/go/worker/keymanager/handler.go @@ -10,7 +10,7 @@ import ( runtimeKeymanager "github.com/oasisprotocol/oasis-core/go/runtime/keymanager/api" "github.com/oasisprotocol/oasis-core/go/runtime/localstorage" workerCommon "github.com/oasisprotocol/oasis-core/go/worker/common" - "github.com/oasisprotocol/oasis-core/go/worker/keymanager/p2p" + committeeCommon "github.com/oasisprotocol/oasis-core/go/worker/common/committee" ) var ( @@ -24,7 +24,7 @@ type hostHandler struct { sync.Mutex w *Worker - remoteClient p2p.Client + remoteClient *committeeCommon.KeyManagerClientWrapper localStorage localstorage.LocalStorage } @@ -48,16 +48,12 @@ func (h *hostHandler) Handle(ctx context.Context, body *protocol.Body) (*protoco switch body.HostRPCCallRequest.Endpoint { case runtimeKeymanager.EnclaveRPCEndpoint: // Call into the remote key manager. - rsp, pf, err := h.remoteClient.CallEnclave(ctx, &p2p.CallEnclaveRequest{ - Data: body.HostRPCCallRequest.Request, - }) + rsp, err := h.remoteClient.CallEnclave(ctx, body.HostRPCCallRequest.Request, body.HostRPCCallRequest.PeerFeedback) if err != nil { return nil, err } - // TODO: Support reporting peer feedback from the enclave. - pf.RecordSuccess() return &protocol.Body{HostRPCCallResponse: &protocol.HostRPCCallResponse{ - Response: cbor.FixSliceForSerde(rsp.Data), + Response: cbor.FixSliceForSerde(rsp), }}, nil default: return nil, errEndpointNotSupported @@ -68,9 +64,13 @@ func (h *hostHandler) Handle(ctx context.Context, body *protocol.Body) (*protoco } func newHostHandler(w *Worker, commonWorker *workerCommon.Worker, localStorage localstorage.LocalStorage) protocol.Handler { + remoteClient := committeeCommon.NewKeyManagerClientWrapper(commonWorker.P2P, commonWorker.Consensus, w.logger) + runtimeID := w.runtime.ID() + remoteClient.SetKeyManagerID(&runtimeID) + return &hostHandler{ w: w, - remoteClient: p2p.NewClient(commonWorker.P2P, commonWorker.Consensus, w.runtime.ID()), + remoteClient: remoteClient, localStorage: localStorage, } } diff --git a/go/worker/keymanager/worker.go b/go/worker/keymanager/worker.go index ed1850f19f9..67287b3a93b 100644 --- a/go/worker/keymanager/worker.go +++ b/go/worker/keymanager/worker.go @@ -750,7 +750,13 @@ func (crw *clientRuntimeWatcher) worker() { select { case <-crw.w.ctx.Done(): return - case <-ch: + case nu := <-ch: + if nu.Reset { + // Ignore reset events to avoid clearing the access list before setting a new one. + // This is safe because a reset event is always followed by a freeze event after the + // nodes have been set (even if the new set is empty). + continue + } crw.w.setAccessList(crw.runtimeID, crw.nodes.GetNodes()) } } diff --git a/runtime/src/enclave_rpc/types.rs b/runtime/src/enclave_rpc/types.rs index ec54afecb34..fd2669ba432 100644 --- a/runtime/src/enclave_rpc/types.rs +++ b/runtime/src/enclave_rpc/types.rs @@ -61,3 +61,11 @@ pub enum Message { Response(Response), Close, } + +/// Feedback on the peer that handled the last EnclaveRPC call. +#[derive(Copy, Clone, Debug, PartialEq, Eq, cbor::Encode, cbor::Decode)] +pub enum PeerFeedback { + Success = 0, + Failure = 1, + BadPeer = 2, +} diff --git a/runtime/src/types.rs b/runtime/src/types.rs index 7a900f8459e..f614d20e7cb 100644 --- a/runtime/src/types.rs +++ b/runtime/src/types.rs @@ -18,6 +18,7 @@ use crate::{ roothash::{self, Block, ComputeResultsHeader, Header}, LightBlock, }, + enclave_rpc, storage::mkvs::{sync, WriteLog}, transaction::types::TxnBatch, }; @@ -204,6 +205,8 @@ pub enum Body { HostRPCCallRequest { endpoint: String, request: Vec, + #[cbor(optional, rename = "pf")] + peer_feedback: Option, }, HostRPCCallResponse { response: Vec,