diff --git a/CHANGELOG.md b/CHANGELOG.md index 70d7c7934062..bb862f4cc274 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## [5.0.0-dev15] + +[5.0.0-dev15]: https://github.com/microsoft/CCF/releases/tag/ccf-5.0.0-dev15 + +### Added + +- CCF now supports a mode where HTTP redirect responses are returned, rather than relying on internal forwarding. This can be used to ensure that write requests are executed on a primary, even if they are initially sent to a backup. This behaviour is enabled by setting the `redirections` field in an `rpc_interface` within cchost's launch config. This can be configured to redirect either directly to a node (if each node has a distinct, accessible name), or to a static load balancer address, depending on the current deployment. See docs for description of [redirection behaviour](https://microsoft.github.io/CCF/main/architecture/request_flow.html#redirection-flow) and [configuration](https://microsoft.github.io/CCF/main/operations/configuration.html#redirections). + ## [5.0.0-dev14] [5.0.0-dev14]: https://github.com/microsoft/CCF/releases/tag/ccf-5.0.0-dev14 diff --git a/CMakeLists.txt b/CMakeLists.txt index 4a02d2287cde..8279107cfa4f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1378,12 +1378,17 @@ if(BUILD_TESTS) ${CMAKE_SOURCE_DIR}/samples/apps/logging/js ) + # This test uses large requests (so too slow for SAN) if(NOT SAN) add_e2e_test( NAME e2e_limits PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/limits.py ) endif() + add_e2e_test( + NAME e2e_redirects PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/redirects.py + ) + add_e2e_test( NAME e2e_logging_http2 PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/e2e_logging.py diff --git a/doc/architecture/request_flow.rst b/doc/architecture/request_flow.rst index 21dc1dc96627..6e979bf65295 100644 --- a/doc/architecture/request_flow.rst +++ b/doc/architecture/request_flow.rst @@ -130,6 +130,65 @@ When follower A receives the forwarded response, it writes this to the TLS sessi NetStackA-->>User: 200 OK "Copied {a} from {A} to {B}" +Redirection flow +---------------- + +CCF supports HTTP redirections as an alternative to forwarding. When a request arrives that cannot be executed locally, rather than forwarding it to an appropriate node over the node-to-node channels, the node can return a HTTP redirect response advising the caller to resubmit the request directly to that node. This uses standard HTTP semantics, reporting the redirect target in a ``Location`` header. Most HTTP clients will have an option to follow this redirect automatically, and all should have an option to enable this behaviour if desired. Alternatively, client applications may choose to intercept this redirect response and manually interpret it, perhaps to alter the resubmitted request or to update the target node for future requests. + +.. warning:: Many HTTP clients will strip out ``Authorization`` headers when following Cross-Origin redirects. This means that if your client is automatically following redirects, and you submit a request with a JWT token as authorization, if you are redirected you may see a surprising authorization failure. In this scenario we recommend intercepting the redirect responses manually, so that the request can be resubmitted without stripping headers. + +Similar to forwarding, the redirect behaviour is partly controlled by per-endpoint metadata, so the initially receiving node must parse the request and go through endpoint dispatch before making a forwarding decision. + +There are currently 2 supported modes for redirections. In the first, the response sends the user directly to the suggested node. This will only work if that node has an accessible name, which can be included in the ``Location`` header and accessed by the user. + +.. mermaid:: + + sequenceDiagram + autonumber + participant U as User + participant B as Backup (nodeA.ccf.com) + participant P as Primary (nodeB.ccf.com) + + U->>B: POST /copy/A/B + B->>B: Lookup endpoint + B->>B: Decide request should be redirected + B->>B: Build redirect response + B-->>U: 307 REDIRECT Location: nodeB.ccf.com/copy/A/B + + U->>P: POST /copy/A/B + P->>P: Lookup endpoint + P->>P: Decide request can be executed + P->>P: Execute request + P-->>U: 200 OK "Copied {a} from {A} to {B}" + +For deployments where nodes are not directly accessible, redirections can still be supported via multiple load balancers. All that is required is `a` public name for each redirect purpose, with up-to-date balancing to the correct nodes. More simply, that currently means maintaining a `write` load balancer which can direct external traffic to a primary. + +.. mermaid:: + + sequenceDiagram + autonumber + participant U as User + participant LB as General LB (service.ccf.com) + participant B as Backup + participant WLB as Write LB (write.service.ccf.com) + participant P as Primary + + U->>LB: POST /copy/A/B + LB->>B: POST /copy/A/B + B->>B: Lookup endpoint + B->>B: Decide request should be redirected + B->>B: Build redirect response + B-->>U: 307 REDIRECT Location: write.service.ccf.com/copy/A/B + + U->>WLB: POST /copy/A/B + WLB->>P: POST /copy/A/B + P->>P: Lookup endpoint + P->>P: Decide request can be executed + P->>P: Execute request + P-->>U: 200 OK "Copied {a} from {A} to {B}" + +To use redirection behaviour, and choose whether to redirect to a node or a load balancer, set the ``redirections`` field in the :doc:`cchost launch configuration `. + External executor flow ---------------------- diff --git a/doc/host_config_schema/cchost_config.json b/doc/host_config_schema/cchost_config.json index ffe1a9f6731a..da7bc037a24e 100644 --- a/doc/host_config_schema/cchost_config.json +++ b/doc/host_config_schema/cchost_config.json @@ -161,6 +161,17 @@ "type": "integer", "default": 3000, "description": "Timeout for forwarded RPC calls (in milliseconds)" + }, + "redirections": { + "type": "object", + "description": "Configure how redirect responses should be produced on this interface. If this is omitted, then forwarding will be used instead", + "properties": { + "to_primary": { + "$ref": "#/$defs/RedirectionResolver", + "description": "Configures how the Location header should be populated, when requests arrive on this interface that must be served by a primary while the receiving node is not a primary" + } + }, + "additionalProperties": false } }, "required": ["bind_address"] @@ -687,5 +698,54 @@ } }, "required": ["enclave", "network", "command"], - "additionalProperties": false + "additionalProperties": false, + "$defs": { + "RedirectionResolver": { + "type": "object", + "properties": { + "kind": { + "enum": ["NodeByRole", "StaticAddress"] + }, + "target": {} + }, + "required": ["kind"], + "if": { + "properties": { + "kind": { + "const": "NodeByRole" + } + } + }, + "then": { + "properties": { + "target": { + "type": "object", + "properties": { + "role": { + "enum": ["primary"], + "default": "primary" + } + }, + "additionalProperties": false + } + } + }, + "else": { + "properties": { + "target": { + "type": "object", + "properties": { + "address": { + "type": "string" + } + }, + "required": ["address"], + "additionalProperties": false + } + }, + "required": ["target"] + }, + "additionalProperties": false + } + } } diff --git a/doc/schemas/gov_openapi.json b/doc/schemas/gov_openapi.json index 119262fea8a9..f869449a7d08 100644 --- a/doc/schemas/gov_openapi.json +++ b/doc/schemas/gov_openapi.json @@ -686,6 +686,27 @@ }, "published_address": { "type": "string" + }, + "redirections": { + "properties": { + "to_primary": { + "properties": { + "kind": { + "enum": [ + "NodeByRole", + "StaticAddress" + ], + "type": "string" + }, + "target": {} + }, + "required": [ + "kind" + ], + "type": "object" + } + }, + "type": "object" } }, "required": [ @@ -788,6 +809,27 @@ }, "published_address": { "type": "string" + }, + "redirections": { + "properties": { + "to_primary": { + "properties": { + "kind": { + "enum": [ + "NodeByRole", + "StaticAddress" + ], + "type": "string" + }, + "target": {} + }, + "required": [ + "kind" + ], + "type": "object" + } + }, + "type": "object" } }, "required": [ @@ -1283,7 +1325,7 @@ "info": { "description": "This API is used to submit and query proposals which affect CCF's public governance tables.", "title": "CCF Governance API", - "version": "4.1.3" + "version": "4.1.4" }, "openapi": "3.0.0", "paths": { diff --git a/doc/schemas/node_openapi.json b/doc/schemas/node_openapi.json index ddf157966562..152e8237d079 100644 --- a/doc/schemas/node_openapi.json +++ b/doc/schemas/node_openapi.json @@ -584,6 +584,9 @@ }, "published_address": { "$ref": "#/components/schemas/string" + }, + "redirections": { + "$ref": "#/components/schemas/NodeInfoNetwork_v2__NetInterface__Redirections" } }, "required": [ @@ -591,6 +594,14 @@ ], "type": "object" }, + "NodeInfoNetwork_v2__NetInterface__Redirections": { + "properties": { + "to_primary": { + "$ref": "#/components/schemas/RedirectionResolverConfig" + } + }, + "type": "object" + }, "NodeMetrics": { "properties": { "sessions": { @@ -686,6 +697,27 @@ ], "type": "string" }, + "RedirectionResolutionKind": { + "enum": [ + "NodeByRole", + "StaticAddress" + ], + "type": "string" + }, + "RedirectionResolverConfig": { + "properties": { + "kind": { + "$ref": "#/components/schemas/RedirectionResolutionKind" + }, + "target": { + "$ref": "#/components/schemas/json" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, "RetirementPhase": { "enum": [ "Ordered", @@ -905,7 +937,7 @@ "info": { "description": "This API provides public, uncredentialed access to service and node state.", "title": "CCF Public Node API", - "version": "4.9.0" + "version": "4.9.1" }, "openapi": "3.0.0", "paths": { diff --git a/include/ccf/endpoint.h b/include/ccf/endpoint.h index 84cb6f938158..3352420e1c0d 100644 --- a/include/ccf/endpoint.h +++ b/include/ccf/endpoint.h @@ -58,6 +58,20 @@ namespace ccf::endpoints Never }; + enum class RedirectionStrategy + { + /** This operation does not need to be redirected, and can be executed on + the receiving node. Most read-only operations can be executed on any + node, so should be marked as None. */ + None, + + /** This operation must be executed on a primary. If the current node is not + a primary, it should attempt to redirect to the primary, or else return + an error. Any write operations must be executed on a primary, so should + be marked as ToPrimary. */ + ToPrimary + }; + enum class Mode { ReadWrite, @@ -77,6 +91,11 @@ namespace ccf::endpoints {ForwardingRequired::Always, "always"}, {ForwardingRequired::Never, "never"}}); + DECLARE_JSON_ENUM( + RedirectionStrategy, + {{RedirectionStrategy::None, "none"}, + {RedirectionStrategy::ToPrimary, "to_primary"}}); + DECLARE_JSON_ENUM( Mode, {{Mode::ReadWrite, "readwrite"}, @@ -106,6 +125,8 @@ namespace ccf::endpoints Mode mode = Mode::ReadWrite; /// Endpoint forwarding policy ForwardingRequired forwarding_required = ForwardingRequired::Always; + /// Endpoint redirection policy + RedirectionStrategy redirection_strategy = RedirectionStrategy::None; /// Authentication policies std::vector authn_policies = {}; /// OpenAPI schema for endpoint @@ -397,6 +418,8 @@ namespace ccf::endpoints */ Endpoint& set_forwarding_required(ForwardingRequired fr); + Endpoint& set_redirection_strategy(RedirectionStrategy rs); + void install(); }; diff --git a/include/ccf/service/node_info_network.h b/include/ccf/service/node_info_network.h index e27f45ba8262..03937ad5eb22 100644 --- a/include/ccf/service/node_info_network.h +++ b/include/ccf/service/node_info_network.h @@ -57,7 +57,28 @@ namespace ccf DECLARE_JSON_REQUIRED_FIELDS( NodeInfoNetwork_v1, rpchost, pubhost, nodehost, nodeport, rpcport, pubport); - static constexpr auto PRIMARY_RPC_INTERFACE = "ccf.default_rpc_interface"; + static constexpr auto PRIMARY_RPC_INTERFACE = "primary_rpc_interface"; + + enum class RedirectionResolutionKind + { + NodeByRole, + StaticAddress + }; + DECLARE_JSON_ENUM( + RedirectionResolutionKind, + {{RedirectionResolutionKind::NodeByRole, "NodeByRole"}, + {RedirectionResolutionKind::StaticAddress, "StaticAddress"}}); + + struct RedirectionResolverConfig + { + RedirectionResolutionKind kind = RedirectionResolutionKind::NodeByRole; + nlohmann::json target; + + bool operator==(const RedirectionResolverConfig&) const = default; + }; + DECLARE_JSON_TYPE_WITH_OPTIONAL_FIELDS(RedirectionResolverConfig); + DECLARE_JSON_REQUIRED_FIELDS(RedirectionResolverConfig, kind); + DECLARE_JSON_OPTIONAL_FIELDS(RedirectionResolverConfig, target); /// Node network information struct NodeInfoNetwork_v2 @@ -94,6 +115,15 @@ namespace ccf /// Timeout for forwarded RPC calls (in milliseconds) std::optional forwarding_timeout_ms = std::nullopt; + struct Redirections + { + RedirectionResolverConfig to_primary; + + bool operator==(const Redirections& other) const = default; + }; + + std::optional redirections = std::nullopt; + bool operator==(const NetInterface& other) const { return bind_address == other.bind_address && @@ -104,7 +134,8 @@ namespace ccf endorsement == other.endorsement && http_configuration == other.http_configuration && accepted_endpoints == other.accepted_endpoints && - forwarding_timeout_ms == other.forwarding_timeout_ms; + forwarding_timeout_ms == other.forwarding_timeout_ms && + redirections == other.redirections; } }; @@ -130,6 +161,11 @@ namespace ccf std::optional acme = std::nullopt; }; + DECLARE_JSON_TYPE_WITH_OPTIONAL_FIELDS( + NodeInfoNetwork_v2::NetInterface::Redirections); + DECLARE_JSON_REQUIRED_FIELDS(NodeInfoNetwork_v2::NetInterface::Redirections); + DECLARE_JSON_OPTIONAL_FIELDS( + NodeInfoNetwork_v2::NetInterface::Redirections, to_primary); DECLARE_JSON_TYPE_WITH_OPTIONAL_FIELDS(NodeInfoNetwork_v2::NetInterface); DECLARE_JSON_REQUIRED_FIELDS(NodeInfoNetwork_v2::NetInterface, bind_address); DECLARE_JSON_OPTIONAL_FIELDS( @@ -142,7 +178,8 @@ namespace ccf app_protocol, http_configuration, accepted_endpoints, - forwarding_timeout_ms); + forwarding_timeout_ms, + redirections); DECLARE_JSON_TYPE(NodeInfoNetwork_v2::ACME); DECLARE_JSON_REQUIRED_FIELDS(NodeInfoNetwork_v2::ACME, configurations); DECLARE_JSON_TYPE_WITH_OPTIONAL_FIELDS(NodeInfoNetwork_v2); @@ -211,7 +248,17 @@ namespace ccf catch (const JsonParseError& jpe) { NodeInfoNetwork_v1 v1; - from_json(j, v1); + try + { + from_json(j, v1); + } + catch (const JsonParseError& _) + { + // If this also fails to parse as a v1, then rethrow the earlier error. + // Configs should now be using v2, and this v1 parsing is just a + // backwards-compatibility shim, which does not get to return errors. + throw jpe; + } nin.node_to_node_interface.bind_address = make_net_address(v1.nodehost, v1.nodeport); diff --git a/samples/config/minimal_config_redirects_role.json b/samples/config/minimal_config_redirects_role.json new file mode 100644 index 000000000000..9d63a56c8c94 --- /dev/null +++ b/samples/config/minimal_config_redirects_role.json @@ -0,0 +1,39 @@ +{ + "enclave": { + "file": "libjs_generic.enclave.so.signed", + "platform": "SGX", + "type": "Release" + }, + "network": { + "node_to_node_interface": { "bind_address": "127.0.0.1:8081" }, + "rpc_interfaces": { + "interface_name": { + "bind_address": "127.0.0.1:8080", + "published_address": "ccf.dummy.com:12345", + "redirections": { + "to_primary": { + "kind": "NodeByRole" + } + } + } + } + }, + "command": { + "type": "Start", + "service_certificate_file": "service_cert.pem", + "start": { + "constitution_files": [ + "validate.js", + "apply.js", + "resolve.js", + "actions.js" + ], + "members": [ + { + "certificate_file": "member0_cert.pem", + "encryption_public_key_file": "member0_enc_pubk.pem" + } + ] + } + } +} diff --git a/samples/config/minimal_config_redirects_static.json b/samples/config/minimal_config_redirects_static.json new file mode 100644 index 000000000000..764d6807f562 --- /dev/null +++ b/samples/config/minimal_config_redirects_static.json @@ -0,0 +1,42 @@ +{ + "enclave": { + "file": "libjs_generic.enclave.so.signed", + "platform": "SGX", + "type": "Release" + }, + "network": { + "node_to_node_interface": { "bind_address": "127.0.0.1:8081" }, + "rpc_interfaces": { + "interface_name": { + "bind_address": "127.0.0.1:8080", + "published_address": "ccf.dummy.com:12345", + "redirections": { + "to_primary": { + "kind": "StaticAddress", + "target": { + "address": "primary.ccf.dummy.com" + } + } + } + } + } + }, + "command": { + "type": "Start", + "service_certificate_file": "service_cert.pem", + "start": { + "constitution_files": [ + "validate.js", + "apply.js", + "resolve.js", + "actions.js" + ], + "members": [ + { + "certificate_file": "member0_cert.pem", + "encryption_public_key_file": "member0_enc_pubk.pem" + } + ] + } + } +} diff --git a/src/endpoints/endpoint.cpp b/src/endpoints/endpoint.cpp index 032fd65e227f..ca2de45a57b3 100644 --- a/src/endpoints/endpoint.cpp +++ b/src/endpoints/endpoint.cpp @@ -66,6 +66,25 @@ namespace ccf::endpoints Endpoint& Endpoint::set_forwarding_required(endpoints::ForwardingRequired fr) { properties.forwarding_required = fr; + + // NB: Should really only override redirection_strategy if it was previously + // implicit, not if it was set explicitly! + switch (properties.forwarding_required) + { + case endpoints::ForwardingRequired::Never: + properties.redirection_strategy = RedirectionStrategy::None; + break; + case endpoints::ForwardingRequired::Sometimes: + case endpoints::ForwardingRequired::Always: + properties.redirection_strategy = RedirectionStrategy::ToPrimary; + break; + } + return *this; + } + + Endpoint& Endpoint::set_redirection_strategy(RedirectionStrategy rs) + { + properties.redirection_strategy = rs; return *this; } diff --git a/src/endpoints/endpoint_registry.cpp b/src/endpoints/endpoint_registry.cpp index ad8e0f7601d7..8e45ad9e0ab5 100644 --- a/src/endpoints/endpoint_registry.cpp +++ b/src/endpoints/endpoint_registry.cpp @@ -214,8 +214,10 @@ namespace ccf::endpoints endpoint.locally_committed_func = default_locally_committed_func; endpoint.authn_policies = ap; - // By default, all write transactions are forwarded + // By default, all transactions are assumed to be writing, and so + // forwarded/redirected endpoint.properties.forwarding_required = ForwardingRequired::Always; + endpoint.properties.redirection_strategy = RedirectionStrategy::ToPrimary; endpoint.installer = this; return endpoint; } @@ -235,7 +237,8 @@ namespace ccf::endpoints f(ro_ctx); }, ap) - .set_forwarding_required(ForwardingRequired::Sometimes); + .set_forwarding_required(ForwardingRequired::Sometimes) + .set_redirection_strategy(RedirectionStrategy::None); } Endpoint EndpointRegistry::make_endpoint_with_local_commit_handler( @@ -270,7 +273,8 @@ namespace ccf::endpoints { return make_endpoint( method, verb, [f](EndpointContext& ctx) { f(ctx); }, ap) - .set_forwarding_required(ForwardingRequired::Sometimes); + .set_forwarding_required(ForwardingRequired::Sometimes) + .set_redirection_strategy(RedirectionStrategy::None); } void EndpointRegistry::install(Endpoint& endpoint) diff --git a/src/node/node_state.h b/src/node/node_state.h index d067f41706a1..2ed39613cc85 100644 --- a/src/node/node_state.h +++ b/src/node/node_state.h @@ -591,7 +591,8 @@ namespace ccf const auto& location = headers.find(http::headers::LOCATION); if ( config.join.follow_redirect && - status == HTTP_STATUS_PERMANENT_REDIRECT && + (status == HTTP_STATUS_PERMANENT_REDIRECT || + status == HTTP_STATUS_TEMPORARY_REDIRECT) && location != headers.end()) { const auto& url = http::parse_url_full(location->second); diff --git a/src/node/rpc/frontend.h b/src/node/rpc/frontend.h index b9863acf77cf..f5dc96e8212a 100644 --- a/src/node/rpc/frontend.h +++ b/src/node/rpc/frontend.h @@ -21,6 +21,7 @@ #include "node/endpoint_context_impl.h" #include "node/node_configuration_subsystem.h" #include "rpc_exception.h" +#include "service/internal_tables_access.h" #define FMT_HEADER_ONLY @@ -198,6 +199,173 @@ namespace ccf return true; } + std::optional resolve_redirect_location( + const RedirectionResolverConfig& resolver, + kv::ReadOnlyTx& tx, + const ccf::ListenInterfaceID& incoming_interface) + { + switch (resolver.kind) + { + case (RedirectionResolutionKind::NodeByRole): + { + const auto role_it = resolver.target.find("role"); + const bool primary = + role_it == resolver.target.end() || role_it.value() == "primary"; + if (!primary) + { + return std::nullopt; + } + + const auto interface_it = resolver.target.find("interface"); + const auto target_interface = + (interface_it == resolver.target.end()) ? + incoming_interface : + interface_it.value().get(); + + const auto primary_id = consensus->primary(); + if (!primary_id.has_value()) + { + return std::nullopt; + } + + const auto nodes = InternalTablesAccess::get_trusted_nodes(tx); + const auto node_it = nodes.find(primary_id.value()); + if (node_it != nodes.end()) + { + const auto& interfaces = node_it->second.rpc_interfaces; + + const auto target_interface_it = interfaces.find(target_interface); + if (target_interface_it != interfaces.end()) + { + return target_interface_it->second.published_address; + } + } + else + { + return std::nullopt; + } + break; + } + + case (RedirectionResolutionKind::StaticAddress): + { + return resolver.target["address"].get(); + break; + } + } + + return std::nullopt; + } + + RedirectionResolverConfig get_redirect_resolver_config( + endpoints::RedirectionStrategy strategy, + const ccf::NodeInfoNetwork_v2::NetInterface::Redirections& redirections) + { + switch (strategy) + { + case (ccf::endpoints::RedirectionStrategy::None): + { + return {}; + } + + case (ccf::endpoints::RedirectionStrategy::ToPrimary): + { + return redirections.to_primary; + } + } + } + + bool check_redirect( + kv::ReadOnlyTx& tx, + std::shared_ptr ctx, + const endpoints::EndpointDefinitionPtr& endpoint, + const ccf::NodeInfoNetwork_v2::NetInterface::Redirections& redirections) + { + auto rs = endpoint->properties.redirection_strategy; + + switch (rs) + { + case (ccf::endpoints::RedirectionStrategy::None): + { + return false; + } + + case (ccf::endpoints::RedirectionStrategy::ToPrimary): + { + const bool is_primary = + (consensus != nullptr) && consensus->can_replicate(); + + // Note: This check is included for parity with forwarding. If we + // should redirect, but can't for fundamental early-node-lifecycle + // reasons, should we try to execute it locally? + const bool redirectable = (consensus != nullptr); + + if (redirectable && !is_primary) + { + auto resolver = get_redirect_resolver_config(rs, redirections); + + const auto listen_interface = + ctx->get_session_context()->interface_id.value_or( + PRIMARY_RPC_INTERFACE); + const auto location = + resolve_redirect_location(resolver, tx, listen_interface); + if (location.has_value()) + { + ctx->set_response_header( + http::headers::LOCATION, + fmt::format( + "https://{}{}", location.value(), ctx->get_request_url())); + ctx->set_response_status(HTTP_STATUS_TEMPORARY_REDIRECT); + return true; + } + + // Should have redirected, but don't know how to. Return an error + ctx->set_error( + HTTP_STATUS_SERVICE_UNAVAILABLE, + ccf::errors::PrimaryNotFound, + "Request should be redirected to primary, but receiving node " + "does not know current primary address"); + return true; + } + return false; + } + + default: + { + LOG_FAIL_FMT("Unhandled redirection strategy: {}", rs); + return false; + } + } + } + + std::optional + get_redirections_config(const ccf::ListenInterfaceID& incoming_interface) + { + if (!node_configuration_subsystem) + { + node_configuration_subsystem = + node_context.get_subsystem(); + if (!node_configuration_subsystem) + { + LOG_FAIL_FMT("Unable to access NodeConfigurationSubsystem"); + return std::nullopt; + } + } + + const auto& node_config_state = node_configuration_subsystem->get(); + const auto& interfaces = + node_config_state.node_config.network.rpc_interfaces; + const auto interface_it = interfaces.find(incoming_interface); + if (interface_it == interfaces.end()) + { + LOG_FAIL_FMT( + "Could not find startup config for interface {}", incoming_interface); + return std::nullopt; + } + + return interface_it->second.redirections; + } + bool check_session_consistency(std::shared_ptr ctx) { if (consensus != nullptr) @@ -458,33 +626,50 @@ namespace ccf return; } - const bool is_primary = (consensus == nullptr) || - consensus->can_replicate() || ctx->is_create_request; - const bool forwardable = (consensus != nullptr); + const auto listen_interface = + ctx->get_session_context()->interface_id.value_or( + PRIMARY_RPC_INTERFACE); + const auto redirections = get_redirections_config(listen_interface); - if (!is_primary && forwardable) + // If a redirections config was specified, then redirections are used + // and no forwarding is done + if (redirections.has_value()) { - switch (endpoint->properties.forwarding_required) + if (check_redirect(*tx_p, ctx, endpoint, redirections.value())) { - case endpoints::ForwardingRequired::Never: - { - break; - } + return; + } + } + else + { + bool is_primary = (consensus == nullptr) || + consensus->can_replicate() || ctx->is_create_request; + const bool forwardable = (consensus != nullptr); - case endpoints::ForwardingRequired::Sometimes: + if (!is_primary && forwardable) + { + switch (endpoint->properties.forwarding_required) { - if (ctx->get_session_context()->is_forwarding) + case endpoints::ForwardingRequired::Never: + { + break; + } + + case endpoints::ForwardingRequired::Sometimes: + { + if (ctx->get_session_context()->is_forwarding) + { + forward(ctx, *tx_p, endpoint); + return; + } + break; + } + + case endpoints::ForwardingRequired::Always: { forward(ctx, *tx_p, endpoint); return; } - break; - } - - case endpoints::ForwardingRequired::Always: - { - forward(ctx, *tx_p, endpoint); - return; } } } diff --git a/src/node/rpc/member_frontend.h b/src/node/rpc/member_frontend.h index 53e9ba76461c..da9831da925a 100644 --- a/src/node/rpc/member_frontend.h +++ b/src/node/rpc/member_frontend.h @@ -575,7 +575,7 @@ namespace ccf openapi_info.description = "This API is used to submit and query proposals which affect CCF's " "public governance tables."; - openapi_info.document_version = "4.1.3"; + openapi_info.document_version = "4.1.4"; } static std::optional get_caller_member_id( diff --git a/src/node/rpc/node_frontend.h b/src/node/rpc/node_frontend.h index 3be81743635d..fc83443b6882 100644 --- a/src/node/rpc/node_frontend.h +++ b/src/node/rpc/node_frontend.h @@ -384,7 +384,7 @@ namespace ccf openapi_info.description = "This API provides public, uncredentialed access to service and node " "state."; - openapi_info.document_version = "4.9.0"; + openapi_info.document_version = "4.9.1"; } void init_handlers() override diff --git a/src/service/internal_tables_access.h b/src/service/internal_tables_access.h index ac69c893a2af..9b1abe268ba7 100644 --- a/src/service/internal_tables_access.h +++ b/src/service/internal_tables_access.h @@ -5,12 +5,15 @@ #include "ccf/crypto/verifier.h" #include "ccf/pal/attestation_sev_snp.h" #include "ccf/service/tables/code_id.h" +#include "ccf/service/tables/constitution.h" #include "ccf/service/tables/members.h" #include "ccf/service/tables/nodes.h" #include "ccf/service/tables/snp_measurements.h" +#include "ccf/service/tables/users.h" #include "ccf/tx.h" #include "node/ledger_secrets.h" #include "node/uvm_endorsements.h" +#include "service/tables/governance_history.h" #include "service/tables/previous_service_identity.h" #include diff --git a/tests/governance_js.py b/tests/governance_js.py index fdb3c7c63395..2d67083504a1 100644 --- a/tests/governance_js.py +++ b/tests/governance_js.py @@ -69,9 +69,24 @@ def set_service_recent_cose_proposals_window_size(proposal_count): ) +def choose_node(network): + # Ideally, this would use find_random_node - you should be able to use any + # node for governance. + # However, many of these tests include a pattern of + # POST /proposal + # GET /proposal + # If the former request is redirected, then the latter may fail (essentially + # reading stale state, assuming session consistency that doesn't exist). + # return network.find_random_node() + + # Instead we ensure that all requests go to the primary + primary, _ = network.find_primary() + return primary + + @reqs.description("Test COSE msg type validation") def test_cose_msg_type_validation(network, args): - node = network.find_random_node() + node = choose_node(network) with node.api_versioned_client( None, None, "member0", api_version=args.gov_api_version @@ -127,7 +142,7 @@ def check_msg_type(verb, path, name, auth_policy): @reqs.description("Test proposal validation") def test_proposal_validation(network, args): - node = network.find_random_node() + node = choose_node(network) def assert_invalid_proposal(proposal_body): try: @@ -213,7 +228,7 @@ def assert_malformed_proposal(proposal_body): @reqs.description("Test proposal storage") def test_proposal_storage(network, args): - node = network.find_random_node() + node = choose_node(network) plausible = os.urandom(32).hex() @@ -253,7 +268,7 @@ def test_proposal_storage(network, args): @reqs.description("Test proposal withdrawal") def test_proposal_withdrawal(network, args): - node = network.find_random_node() + node = choose_node(network) infra.clients.get_clock().advance() plausible = os.urandom(32).hex() @@ -308,7 +323,7 @@ def test_proposal_withdrawal(network, args): @reqs.description("Test ballot storage and validation") def test_ballot_storage(network, args): - node = network.find_random_node() + node = choose_node(network) infra.clients.get_clock().advance() @@ -371,7 +386,7 @@ def test_ballot_storage(network, args): @reqs.description("Test pure proposals") def test_pure_proposals(network, args): - node = network.find_random_node() + node = choose_node(network) with node.api_versioned_client( None, None, "member0", api_version=args.gov_api_version @@ -402,7 +417,7 @@ def test_pure_proposals(network, args): @reqs.description("Test proposal replay protection") def test_proposal_replay_protection(network, args): - node = network.find_random_node() + node = choose_node(network) with node.api_versioned_client( None, None, "member0", api_version=args.gov_api_version @@ -479,7 +494,7 @@ def test_proposal_replay_protection(network, args): @reqs.description("Test open proposals") def test_all_open_proposals(network, args): - node = network.find_random_node() + node = choose_node(network) with node.api_versioned_client( None, None, "member0", api_version=args.gov_api_version ) as c: @@ -526,7 +541,7 @@ def opposite(js_bool): @reqs.description("Test vote proposals") def test_proposals_with_votes(network, args): - node = network.find_random_node() + node = choose_node(network) with node.api_versioned_client( None, None, "member0", api_version=args.gov_api_version ) as c: @@ -600,7 +615,7 @@ def test_proposals_with_votes(network, args): @reqs.description("Test vote failure reporting") def test_vote_failure_reporting(network, args): - node = network.find_random_node() + node = choose_node(network) error_body = f"Sample error ({uuid.uuid4()})" @@ -643,7 +658,7 @@ def test_vote_failure_reporting(network, args): @reqs.description("Test operator proposals and votes") def test_operator_proposals_and_votes(network, args): - node = network.find_random_node() + node = choose_node(network) with node.api_versioned_client( None, None, "member0", api_version=args.gov_api_version ) as c: @@ -671,7 +686,7 @@ def test_operator_proposals_and_votes(network, args): @reqs.description("Test operator provisioner proposals") def test_operator_provisioner_proposals_and_votes(network, args): - node = network.find_random_node() + node = choose_node(network) def propose_and_assert_accepted(signer_id, proposal): with node.api_versioned_client( @@ -751,7 +766,7 @@ def propose_and_assert_accepted(signer_id, proposal): @reqs.description("Test actions") def test_actions(network, args): - node = network.find_random_node() + node = choose_node(network) # Rekey ledger network.consortium.trigger_ledger_rekey(node) @@ -857,7 +872,7 @@ def test_actions(network, args): @reqs.description("Test resolve and apply failures") def test_apply(network, args): - node = network.find_random_node() + node = choose_node(network) with node.api_versioned_client( None, None, "member0", api_version=args.gov_api_version @@ -921,7 +936,7 @@ def test_apply(network, args): @reqs.description("Test set_constitution") def test_set_constitution(network, args): - node = network.find_random_node() + node = choose_node(network) infra.clients.get_clock().advance() # Create some open proposals diff --git a/tests/infra/clients.py b/tests/infra/clients.py index b108ac02d0f1..cecd27413622 100644 --- a/tests/infra/clients.py +++ b/tests/infra/clients.py @@ -155,8 +155,6 @@ class Request: http_verb: str #: HTTP headers headers: dict - #: Whether redirect headers should be transparently followed - allow_redirects: bool def __str__(self): string = f"{self.http_verb} {self.path}" @@ -263,7 +261,11 @@ class Response: def __str__(self): versioned = (self.view, self.seqno) != (None, None) - status_color = "red" if self.status_code // 100 in (4, 5) else "green" + status_category = self.status_code // 100 + redirect = status_category == 3 + status_color = ( + "red" if status_category in (4, 5) else "yellow" if redirect else "green" + ) body_s = escape_loguru_tags(truncate(str(self.body))) # Body can't end with a \, or it will escape the loguru closing tag if len(body_s) > 0 and body_s[-1] == "\\": @@ -271,6 +273,11 @@ def __str__(self): return ( f"<{status_color}>{self.status_code} " + + ( + f"[Redirect to -> {self.headers['location']}] " + if redirect + else "" + ) + (f"@{self.view}.{self.seqno} " if versioned else "") + f"{body_s}" ) @@ -485,9 +492,6 @@ def request( cmd += [url, "-X", request.http_verb, "-i", f"-m {timeout}"] - if request.allow_redirects: - cmd.append("-L") - headers = {} if self.common_headers is not None: headers.update(self.common_headers) @@ -546,9 +550,6 @@ def request( cmd += [url, "-X", request.http_verb, "-i", f"-m {timeout}"] - if request.allow_redirects: - cmd.append("-L") - if self.cose_signing_auth: cmd.extend(["--data-binary", "@-"]) else: @@ -755,7 +756,6 @@ def request( url=f"{self.protocol}://{self.hostname}{request.path}", auth=auth, headers=extra_headers, - follow_redirects=request.allow_redirects, timeout=timeout, content=request_body, ) @@ -803,7 +803,7 @@ class RawSocketClient: def __init__( self, - netloc: str, + hostname: str, ca: str, session_auth: Optional[Identity] = None, signing_auth: Optional[Identity] = None, @@ -833,7 +833,7 @@ def __init__( else: self.signing_details = None - hostname, port = infra.interfaces.split_netloc(netloc) + hostname, port = infra.interfaces.split_netloc(hostname) self.socket = RawSocketClient._create_socket( hostname, @@ -948,31 +948,6 @@ def request( ) response = Response.from_socket(self.socket) - while response.status_code == 308 and request.allow_redirects: - assert ( - self.signing_details is None - ), f"Received redirect response from {request.path}, but submitted signed request. Combination of signed requests and forwarding is currently unsupported" - - # Create a temporary socket to follow this redirect - redirect_url = response.headers["location"] - LOG.trace(f"Following redirect to: {redirect_url}") - parsed = urllib.parse.urlparse(redirect_url) - with RawSocketClient._create_socket( - parsed.hostname, - parsed.port, - self.ca, - self.session_auth, - ) as redirect_socket: - redirect_socket.settimeout(timeout) - RawSocketClient._send_request( - ssl_socket=redirect_socket, - verb=request.http_verb, - path=parsed.path, - headers=extra_headers, - content=request_body, - ) - response = Response.from_socket(redirect_socket) - return response def close(self): @@ -1044,15 +1019,15 @@ def __init__( self.sign = bool(signing_auth) self.cose = bool(cose_signing_auth) - self.client_impl = impl_type( - self.hostname, - ca, - session_auth, - signing_auth, - cose_signing_auth, - common_headers, + self.client_args = { + "ca": ca, + "session_auth": session_auth, + "signing_auth": signing_auth, + "cose_signing_auth": cose_signing_auth, + "common_headers": common_headers, **kwargs, - ) + } + self.client_impl = impl_type(hostname=self.hostname, **self.client_args) def _response(self, response: Response) -> Response: LOG.info(response) @@ -1071,10 +1046,44 @@ def _call( ) -> Response: if headers is None: headers = {} - r = Request(path, body, http_verb, headers, allow_redirects) + + r = Request(path, body, http_verb, headers) flush_info([f"{self.description} {r}"], log_capture, 3) + response = self.client_impl.request(r, timeout, cose_header_parameters_override) flush_info([str(response)], log_capture, 3) + + # NB: We follow redirects at this level, because we do not trust the underlying + # client implementation to do so without modifying the request (eg - removing + # the Authorization header) + redirect_count = 0 + while ( + allow_redirects + and redirect_count < 20 + and (response.status_code == 308 or response.status_code == 307) + ): + redirect_count += 1 + assert ( + "location" in response.headers + ), f"Received redirect response without location header: {response}" + + redirect_url = response.headers["location"] + split = urllib.parse.urlsplit(redirect_url) + hostname = split.netloc or self.hostname + redirect_path = urllib.parse.urlunsplit(("", "", *split[2:])) + + # Construct a temporary client to follow this redirect + temp_client = type(self.client_impl)(hostname=hostname, **self.client_args) + + # Copy any test-specific decorators from the main client to the temporary client + temp_client._corrupt_signature = self.client_impl._corrupt_signature + temp_client.cose_header_builder = self.client_impl.cose_header_builder + + r = Request(redirect_path, body, http_verb, headers) + + response = temp_client.request(r, timeout, cose_header_parameters_override) + flush_info([str(response)], log_capture, 3) + return response def call( diff --git a/tests/infra/e2e_args.py b/tests/infra/e2e_args.py index 7e909fbbcc29..e7b94aa4426f 100644 --- a/tests/infra/e2e_args.py +++ b/tests/infra/e2e_args.py @@ -415,7 +415,6 @@ def cli_args( type=str, default=infra.clients.API_VERSION_PREVIEW_01, ) - add(parser) if accept_unknown: diff --git a/tests/infra/interfaces.py b/tests/infra/interfaces.py index 40fb46377406..6041ac051f81 100644 --- a/tests/infra/interfaces.py +++ b/tests/infra/interfaces.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the Apache 2.0 License. -from dataclasses import dataclass +from dataclasses import dataclass, asdict from typing import Optional, Dict from enum import Enum, auto import urllib.parse @@ -75,6 +75,63 @@ class Interface: port: int = 0 +class RedirectionResolver: + pass + + +@dataclass +class NodeByRoleResolver(RedirectionResolver): + kind: str = "NodeByRole" + target = {"role": "primary"} + + @staticmethod + def to_json(nbrr): + return asdict(nbrr) + + @staticmethod + def from_json(json): + nbrr = NodeByRoleResolver() + nbrr.target = json["target"] + return nbrr + + +class StaticAddressResolver(RedirectionResolver): + kind: str = "StaticAddress" + target_address: str + + def __init__(self, address): + self.target_address = address + + @staticmethod + def to_json(sar): + return { + "kind": sar.kind, + "target": {"address": sar.target_address}, + } + + @staticmethod + def from_json(json): + sar = StaticAddressResolver() + sar.target_address = json["target"]["address"] + return sar + + +@dataclass +class RedirectionConfig: + to_primary: RedirectionResolver = NodeByRoleResolver() + + @staticmethod + def to_json(rc): + return {"to_primary": rc.to_primary.to_json(rc.to_primary)} + + @staticmethod + def from_json(json): + if json["kind"] == "NodeByRole": + return NodeByRoleResolver.from_json(json) + elif json["kind"] == "StaticAddress": + return StaticAddressResolver.from_json(json) + + @dataclass class RPCInterface(Interface): # How nodes are created (local, ssh, ...) @@ -97,6 +154,7 @@ class RPCInterface(Interface): acme_configuration: Optional[str] = None accepted_endpoints: Optional[str] = None forwarding_timeout_ms: Optional[int] = None + redirections: Optional[RedirectionConfig] = None app_protocol: str = "HTTP1" @staticmethod @@ -154,6 +212,8 @@ def to_json(interface): r["accepted_endpoints"] = interface.accepted_endpoints if interface.forwarding_timeout_ms: r["forwarding_timeout_ms"] = interface.forwarding_timeout_ms + if interface.redirections: + r["redirections"] = RedirectionConfig.to_json(interface.redirections) return r @staticmethod @@ -178,6 +238,8 @@ def from_json(json): interface.forwarding_timeout_ms = json.get( "forwarding_timeout_ms", DEFAULT_FORWARDING_TIMEOUT_MS ) + if "redirections" in json: + interface.redirections = RedirectionConfig.from_json(json["redirections"]) if "endorsement" in json: interface.endorsement = Endorsement.from_json(json["endorsement"]) interface.accepted_endpoints = json.get("accepted_endpoints") diff --git a/tests/infra/member.py b/tests/infra/member.py index 9820c750997c..3897f23494bd 100644 --- a/tests/infra/member.py +++ b/tests/infra/member.py @@ -402,6 +402,9 @@ def get_and_submit_recovery_share(self, remote_node): "--cacert", os.path.join(self.common_dir, "service_cert.pem"), ] + + cmd += ["-L"] + res = infra.proc.ccall( *cmd, log_output=True, diff --git a/tests/memberclient.py b/tests/memberclient.py index 546fed241903..aa5e0f15cbc4 100644 --- a/tests/memberclient.py +++ b/tests/memberclient.py @@ -16,7 +16,7 @@ @reqs.description("Send an unsigned request where signature is required") def test_missing_signature_header(network, args): - node = network.find_node_by_role() + node = network.find_node_by_role(role=infra.network.NodeRole.PRIMARY) member = network.consortium.get_any_active_member() # NB: This client uses member cert auth, so no signature is inserted later with node.client(member.local_id) as mc: @@ -81,7 +81,7 @@ def modified_signature(request): @reqs.description("Send a corrupted signature where signed request is required") def test_corrupted_signature(network, args): - node = network.find_node_by_role() + node = network.find_node_by_role(role=infra.network.NodeRole.PRIMARY) # Test each supported curve for curve in infra.network.EllipticCurve: @@ -112,8 +112,7 @@ def test_corrupted_signature(network, args): @reqs.description("Test various governance operations") def test_governance(network, args): - node = network.find_node_by_role() - primary, _ = network.find_primary() + node = network.find_node_by_role(role=infra.network.NodeRole.PRIMARY) LOG.info("Original members can ACK") network.consortium.get_any_active_member().ack(node) @@ -127,7 +126,7 @@ def test_governance(network, args): try: proposal = network.consortium.get_any_active_member().propose( - primary, unkwown_proposal + node, unkwown_proposal ) assert False, "Unknown proposal should fail on validation" except infra.proposal.ProposalNotCreated: @@ -144,7 +143,7 @@ def test_governance(network, args): ) LOG.info("Check proposal has been recorded in open state") - proposal = network.consortium.get_proposal(primary, new_member_proposal.proposal_id) + proposal = network.consortium.get_proposal(node, new_member_proposal.proposal_id) assert proposal.state == infra.proposal.ProposalState.OPEN LOG.info("Rest of consortium accept the proposal") @@ -217,7 +216,7 @@ def test_governance(network, args): assert response.status_code == http.HTTPStatus.OK.value assert proposal.state == infra.proposal.ProposalState.WITHDRAWN - proposal = network.consortium.get_proposal(primary, proposal.proposal_id) + proposal = network.consortium.get_proposal(node, proposal.proposal_id) assert proposal.state == infra.proposal.ProposalState.WITHDRAWN if new_member.gov_api_impl.API_VERSION == infra.clients.API_VERSION_CLASSIC: diff --git a/tests/partitions_test.py b/tests/partitions_test.py index c0bfdd59add7..39be4b0985d1 100644 --- a/tests/partitions_test.py +++ b/tests/partitions_test.py @@ -734,7 +734,8 @@ def run(args): test_isolate_and_reconnect_primary(network, args, iteration=n) test_election_reconfiguration(network, args) test_forwarding_timeout(network, args) - if not args.http2: # HTTP2 doesn't support forwarding + # HTTP2 doesn't support forwarding + if not args.http2: test_session_consistency(network, args) test_ledger_invariants(network, args) diff --git a/tests/redirects.py b/tests/redirects.py new file mode 100644 index 000000000000..ac1d799fca68 --- /dev/null +++ b/tests/redirects.py @@ -0,0 +1,151 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the Apache 2.0 License. +import infra.network +import infra.e2e_args +import infra.interfaces +import infra.net +from infra.runner import ConcurrentRunner +import http +import time + +from loguru import logger as LOG + + +def test_redirects_with_node_role_config(network, args): + paths = ("/app/log/private", "/app/log/public") + msg = "Redirect test" + + def test_redirect_to_node(talk_to, redirect_to): + interface = redirect_to.host.rpc_interfaces[ + infra.interfaces.PRIMARY_RPC_INTERFACE + ] + loc = f"https://{interface.public_host}:{interface.public_port}" + + with talk_to.client("user0") as c: + for path in paths: + r = c.post(path, {"id": 42, "msg": msg}, allow_redirects=False) + assert r.status_code == http.HTTPStatus.TEMPORARY_REDIRECT.value + assert "location" in r.headers + assert r.headers["location"] == f"{loc}{path}", r.headers + + LOG.info("Redirect to original primary") + primary, orig_backups = network.find_nodes() + for backup in orig_backups: + test_redirect_to_node(backup, primary) + + LOG.info("Redirect to subsequent primary") + primary.stop() + network.wait_for_new_primary(primary) + new_primary, new_backups = network.find_nodes() + for backup in new_backups: + test_redirect_to_node(backup, new_primary) + + LOG.info("Subsequent primary no longer redirects") + assert new_primary in orig_backups # Check it WAS a backup + with new_primary.client("user0") as c: + for path in paths: + r = c.post(path, {"id": 42, "msg": msg}, allow_redirects=False) + assert r.status_code == http.HTTPStatus.OK.value + + LOG.info("Redirects fail when no primary available") + new_primary.stop() + backup = new_backups[0] + start_time = time.time() + timeout = network.observed_election_duration + end_time = start_time + timeout + while time.time() < end_time: + with backup.client() as c: + r = c.head("/node/primary", allow_redirects=False) + if r.status_code == http.HTTPStatus.INTERNAL_SERVER_ERROR: + break + time.sleep(0.5) + else: + raise TimeoutError(f"Node failed to recognise primary death after {timeout}s") + + with backup.client("user0") as c: + for path in paths: + r = c.post(path, {"id": 42, "msg": msg}, allow_redirects=False) + assert r.status_code == http.HTTPStatus.SERVICE_UNAVAILABLE.value + assert r.body.json()["error"]["code"] == "PrimaryNotFound" + + +def test_redirects_with_static_name_config(network, args): + hostname = "primary.my.ccf.service.example.test" + + paths = ("/app/log/private", "/app/log/public") + msg = "Redirect test" + + new_node = network.create_node( + infra.interfaces.HostSpec( + rpc_interfaces={ + infra.interfaces.PRIMARY_RPC_INTERFACE: infra.interfaces.RPCInterface( + host=infra.net.expand_localhost(), + redirections=infra.interfaces.RedirectionConfig( + to_primary=infra.interfaces.StaticAddressResolver(hostname) + ), + ) + } + ) + ) + network.join_node(new_node, args.package, args) + network.trust_node(new_node, args) + + with new_node.client("user0") as c: + for path in paths: + r = c.post(path, {"id": 42, "msg": msg}, allow_redirects=False) + assert r.status_code == http.HTTPStatus.TEMPORARY_REDIRECT.value + assert "location" in r.headers + assert r.headers["location"] == f"https://{hostname}{path}", r.headers + + +def run_redirect_tests_role(args): + for node in args.nodes: + primary_interface = node.rpc_interfaces[infra.interfaces.PRIMARY_RPC_INTERFACE] + primary_interface.redirections = infra.interfaces.RedirectionConfig( + to_primary=infra.interfaces.NodeByRoleResolver() + ) + + with infra.network.network( + args.nodes, + args.binary_dir, + args.debug_nodes, + args.perf_nodes, + pdb=args.pdb, + ) as network: + network.start_and_open(args) + + test_redirects_with_node_role_config(network, args) + # ^ This test kills nodes, so be careful if you follow it! + + +def run_redirect_tests_static(args): + with infra.network.network( + args.nodes, + args.binary_dir, + args.debug_nodes, + args.perf_nodes, + pdb=args.pdb, + ) as network: + network.start_and_open(args) + + test_redirects_with_static_name_config(network, args) + + +if __name__ == "__main__": + cr = ConcurrentRunner() + + cr.add( + "redirects_role", + run_redirect_tests_role, + package="samples/apps/logging/liblogging", + nodes=infra.e2e_args.min_nodes(cr.args, f=1), + ) + + cr.add( + "redirects_static", + run_redirect_tests_static, + package="samples/apps/logging/liblogging", + nodes=infra.e2e_args.min_nodes(cr.args, f=0), + ) + + cr.run()